blob: dc164e7b61d178aaede70bf482544107bb0d7b5a [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010022#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010023#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020024#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010026#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010027#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080028#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020030#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020031#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010032#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010033#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020034#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020035#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020036#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010037#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020038#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010039#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020040#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070041#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020043#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 18:47:52 +010044#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010045#include "modules/video_coding/codecs/h264/include/h264.h"
46#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
47#include "modules/video_coding/codecs/vp8/include/vp8.h"
48#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020049#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020050#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020051#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010052#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020053#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010054#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020055#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080057#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020058#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020059#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/encoder_settings.h"
61#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010065#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010066#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020067#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090068#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020069#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010070#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020071#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070072
73namespace webrtc {
74
sprang57c2fff2017-01-16 06:24:02 -080075using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Ge;
80using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010081using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020082using ::testing::Le;
83using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010085using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010089using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020090using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080091
perkj803d97f2016-11-01 11:45:46 -070092namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020093const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010094const int kQpLow = 1;
95const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020096const int kMinFramerateFps = 2;
97const int kMinBalancedFramerateFps = 7;
98const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080099const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200100const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
101const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
102const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
103const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800104const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700105const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200106const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200107const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800112
Asa Persson606d3cb2021-10-04 10:07:11 +0200113uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200114 0x00, 0x00, 0x03, 0x03, 0xF4,
115 0x05, 0x03, 0xC7, 0xE0, 0x1B,
116 0x41, 0x10, 0x8D, 0x00};
117
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100118const uint8_t kCodedFrameVp8Qp25[] = {
119 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
120 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
121 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
122
Markus Handell818e7fb2021-12-30 13:01:33 +0100123VideoFrame CreateSimpleNV12Frame() {
124 return VideoFrame::Builder()
125 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
126 /*width=*/16, /*height=*/16))
127 .build();
128}
129
Markus Handell8d87c462021-12-16 11:37:16 +0100130void PassAFrame(
131 TaskQueueBase* encoder_queue,
132 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
133 int64_t ntp_time_ms) {
134 encoder_queue->PostTask(
135 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 13:01:33 +0100136 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
137 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 11:37:16 +0100138 }));
139}
140
perkj803d97f2016-11-01 11:45:46 -0700141class TestBuffer : public webrtc::I420Buffer {
142 public:
143 TestBuffer(rtc::Event* event, int width, int height)
144 : I420Buffer(width, height), event_(event) {}
145
146 private:
147 friend class rtc::RefCountedObject<TestBuffer>;
148 ~TestBuffer() override {
149 if (event_)
150 event_->Set();
151 }
152 rtc::Event* const event_;
153};
154
Henrik Boström56db9ff2021-03-24 09:06:45 +0100155// A fake native buffer that can't be converted to I420. Upon scaling, it
156// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700157class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
158 public:
159 FakeNativeBuffer(rtc::Event* event, int width, int height)
160 : event_(event), width_(width), height_(height) {}
161 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
162 int width() const override { return width_; }
163 int height() const override { return height_; }
164 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
165 return nullptr;
166 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100167 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
168 int offset_x,
169 int offset_y,
170 int crop_width,
171 int crop_height,
172 int scaled_width,
173 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200174 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
175 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100176 }
Noah Richards51db4212019-06-12 06:59:12 -0700177
178 private:
179 friend class rtc::RefCountedObject<FakeNativeBuffer>;
180 ~FakeNativeBuffer() override {
181 if (event_)
182 event_->Set();
183 }
184 rtc::Event* const event_;
185 const int width_;
186 const int height_;
187};
188
Evan Shrubsole895556e2020-10-05 09:15:13 +0200189// A fake native buffer that is backed by an NV12 buffer.
190class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
191 public:
192 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
193 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
194
195 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
196 int width() const override { return nv12_buffer_->width(); }
197 int height() const override { return nv12_buffer_->height(); }
198 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
199 return nv12_buffer_->ToI420();
200 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200201 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
202 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
203 if (absl::c_find(types, Type::kNV12) != types.end()) {
204 return nv12_buffer_;
205 }
206 return nullptr;
207 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200208 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
209
210 private:
211 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
212 ~FakeNV12NativeBuffer() override {
213 if (event_)
214 event_->Set();
215 }
216 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
217 rtc::Event* const event_;
218};
219
Niels Möller7dc26b72017-12-06 10:27:48 +0100220class CpuOveruseDetectorProxy : public OveruseFrameDetector {
221 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200222 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
223 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200224 last_target_framerate_fps_(-1),
225 framerate_updated_event_(true /* manual_reset */,
226 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100227 virtual ~CpuOveruseDetectorProxy() {}
228
229 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200230 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 last_target_framerate_fps_ = framerate_fps;
232 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200233 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100234 }
235
236 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200237 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100238 return last_target_framerate_fps_;
239 }
240
Niels Möller4db138e2018-04-19 09:04:13 +0200241 CpuOveruseOptions GetOptions() { return options_; }
242
Henrik Boström381d1092020-05-12 18:49:07 +0200243 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
244
Niels Möller7dc26b72017-12-06 10:27:48 +0100245 private:
Markus Handella3765182020-07-08 13:13:32 +0200246 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100247 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200248 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100249};
250
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200251class FakeVideoSourceRestrictionsListener
252 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200253 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200254 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200255 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200256 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200257 RTC_DCHECK(was_restrictions_updated_);
258 }
259
260 rtc::Event* restrictions_updated_event() {
261 return &restrictions_updated_event_;
262 }
263
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200264 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200265 void OnVideoSourceRestrictionsUpdated(
266 VideoSourceRestrictions restrictions,
267 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200268 rtc::scoped_refptr<Resource> reason,
269 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200270 was_restrictions_updated_ = true;
271 restrictions_updated_event_.Set();
272 }
273
274 private:
275 bool was_restrictions_updated_;
276 rtc::Event restrictions_updated_event_;
277};
278
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279auto WantsFps(Matcher<int> fps_matcher) {
280 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
281 fps_matcher);
282}
283
284auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
285 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
286 AllOf(max_pixel_matcher, Gt(0)));
287}
288
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200289auto ResolutionMax() {
290 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200291 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200292 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
293 Eq(absl::nullopt)));
294}
295
296auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200297 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200298}
299
300auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200301 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200302}
303
304auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200305 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200306}
307
308auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200310}
311
312auto FpsMaxResolutionMax() {
313 return AllOf(FpsMax(), ResolutionMax());
314}
315
316auto UnlimitedSinkWants() {
317 return AllOf(FpsUnlimited(), ResolutionMax());
318}
319
320auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
321 Matcher<int> fps_range_matcher;
322
323 if (last_frame_pixels <= 320 * 240) {
324 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200325 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200326 fps_range_matcher = AllOf(Ge(10), Le(15));
327 } else if (last_frame_pixels <= 640 * 480) {
328 fps_range_matcher = Ge(15);
329 } else {
330 fps_range_matcher = Eq(kDefaultFramerate);
331 }
332 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
333 fps_range_matcher);
334}
335
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200336auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
337 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
338 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
339}
340
341auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
342 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
343}
344
345auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
346 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
347}
348
349auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
351 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
352}
353
354auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
355 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
356 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
357}
358
359auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
360 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
361 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
362}
363
364auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
365 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
366 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
367}
368
mflodmancc3d4422017-08-03 08:27:51 -0700369class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700370 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100371 VideoStreamEncoderUnderTest(
372 TimeController* time_controller,
373 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
374 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
375 encoder_queue,
376 SendStatisticsProxy* stats_proxy,
377 const VideoStreamEncoderSettings& settings,
378 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100379 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200380 const FieldTrialsView& field_trials,
381 int num_cores)
Markus Handell9a478b52021-11-18 16:07:01 +0100382 : VideoStreamEncoder(time_controller->GetClock(),
Erik Språnge4589cb2022-04-06 16:44:30 +0200383 num_cores,
Markus Handell9a478b52021-11-18 16:07:01 +0100384 stats_proxy,
385 settings,
386 std::unique_ptr<OveruseFrameDetector>(
387 overuse_detector_proxy_ =
388 new CpuOveruseDetectorProxy(stats_proxy)),
389 std::move(cadence_adapter),
390 std::move(encoder_queue),
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100391 allocation_callback_type,
392 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200393 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200394 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200395 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200396 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200397 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200398 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200399 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100401 }
perkj803d97f2016-11-01 11:45:46 -0700402
Henrik Boström381d1092020-05-12 18:49:07 +0200403 void SetSourceAndWaitForRestrictionsUpdated(
404 rtc::VideoSourceInterface<VideoFrame>* source,
405 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200406 FakeVideoSourceRestrictionsListener listener;
407 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200408 SetSource(source, degradation_preference);
409 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200410 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200411 }
412
413 void SetSourceAndWaitForFramerateUpdated(
414 rtc::VideoSourceInterface<VideoFrame>* source,
415 const DegradationPreference& degradation_preference) {
416 overuse_detector_proxy_->framerate_updated_event()->Reset();
417 SetSource(source, degradation_preference);
418 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
419 }
420
421 void OnBitrateUpdatedAndWaitForManagedResources(
422 DataRate target_bitrate,
423 DataRate stable_target_bitrate,
424 DataRate link_allocation,
425 uint8_t fraction_lost,
426 int64_t round_trip_time_ms,
427 double cwnd_reduce_ratio) {
428 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
429 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
430 // Bitrate is updated on the encoder queue.
431 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200432 }
433
kthelgason2fc52542017-03-03 00:24:41 -0800434 // This is used as a synchronisation mechanism, to make sure that the
435 // encoder queue is not blocked before we start sending it frames.
436 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100437 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800438 }
439
Henrik Boström91aa7322020-04-28 12:24:33 +0200440 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200441 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200442 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200443 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200444 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 event.Set();
446 });
447 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100448 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 void TriggerCpuUnderuse() {
452 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200453 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200454 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 event.Set();
456 });
457 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100458 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200459 }
kthelgason876222f2016-11-29 01:44:11 -0800460
Henrik Boström91aa7322020-04-28 12:24:33 +0200461 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200462 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200463 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200464 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200465 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 event.Set();
467 });
468 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100469 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200470 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200471 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200473 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200474 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200475 event.Set();
476 });
477 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100478 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200479 }
480
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200481 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100482 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200483 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
484 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200485 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700486};
487
Noah Richards51db4212019-06-12 06:59:12 -0700488// Simulates simulcast behavior and makes highest stream resolutions divisible
489// by 4.
490class CroppingVideoStreamFactory
491 : public VideoEncoderConfig::VideoStreamFactoryInterface {
492 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200493 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700494
495 private:
496 std::vector<VideoStream> CreateEncoderStreams(
497 int width,
498 int height,
499 const VideoEncoderConfig& encoder_config) override {
500 std::vector<VideoStream> streams = test::CreateVideoStreams(
501 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700502 return streams;
503 }
Noah Richards51db4212019-06-12 06:59:12 -0700504};
505
sprangb1ca0732017-02-01 08:38:12 -0800506class AdaptingFrameForwarder : public test::FrameForwarder {
507 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200508 explicit AdaptingFrameForwarder(TimeController* time_controller)
509 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700510 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800511
512 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200513 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800514 adaptation_enabled_ = enabled;
515 }
516
asaperssonfab67072017-04-04 05:51:49 -0700517 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200518 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800519 return adaptation_enabled_;
520 }
521
Henrik Boström1124ed12021-02-25 10:30:39 +0100522 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
523 // the resolution or frame rate was different than it is currently. If
524 // something else is modified, such as encoder resolutions, but the resolution
525 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700526 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200527 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700528 return last_wants_;
529 }
530
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200531 absl::optional<int> last_sent_width() const { return last_width_; }
532 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800533
sprangb1ca0732017-02-01 08:38:12 -0800534 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200535 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100536 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200537
sprangb1ca0732017-02-01 08:38:12 -0800538 int cropped_width = 0;
539 int cropped_height = 0;
540 int out_width = 0;
541 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700542 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000543 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
544 << "w=" << video_frame.width()
545 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700546 if (adapter_.AdaptFrameResolution(
547 video_frame.width(), video_frame.height(),
548 video_frame.timestamp_us() * 1000, &cropped_width,
549 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100550 VideoFrame adapted_frame =
551 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200552 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100553 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200554 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100555 .set_timestamp_ms(99)
556 .set_rotation(kVideoRotation_0)
557 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100558 if (video_frame.has_update_rect()) {
559 adapted_frame.set_update_rect(
560 video_frame.update_rect().ScaleWithFrame(
561 video_frame.width(), video_frame.height(), 0, 0,
562 video_frame.width(), video_frame.height(), out_width,
563 out_height));
564 }
sprangc5d62e22017-04-02 23:53:04 -0700565 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800566 last_width_.emplace(adapted_frame.width());
567 last_height_.emplace(adapted_frame.height());
568 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200569 last_width_ = absl::nullopt;
570 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700571 }
sprangb1ca0732017-02-01 08:38:12 -0800572 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000573 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800574 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800575 last_width_.emplace(video_frame.width());
576 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800577 }
578 }
579
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200580 void OnOutputFormatRequest(int width, int height) {
581 absl::optional<std::pair<int, int>> target_aspect_ratio =
582 std::make_pair(width, height);
583 absl::optional<int> max_pixel_count = width * height;
584 absl::optional<int> max_fps;
585 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
586 max_fps);
587 }
588
sprangb1ca0732017-02-01 08:38:12 -0800589 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
590 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200591 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100592 rtc::VideoSinkWants prev_wants = sink_wants_locked();
593 bool did_adapt =
594 prev_wants.max_pixel_count != wants.max_pixel_count ||
595 prev_wants.target_pixel_count != wants.target_pixel_count ||
596 prev_wants.max_framerate_fps != wants.max_framerate_fps;
597 if (did_adapt) {
598 last_wants_ = prev_wants;
599 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100600 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200601 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800602 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200603
604 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800605 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200606 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
607 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200608 absl::optional<int> last_width_;
609 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800610};
sprangc5d62e22017-04-02 23:53:04 -0700611
Niels Möller213618e2018-07-24 09:29:58 +0200612// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700613class MockableSendStatisticsProxy : public SendStatisticsProxy {
614 public:
615 MockableSendStatisticsProxy(Clock* clock,
616 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100617 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200618 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100619 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700620
621 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200622 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700623 if (mock_stats_)
624 return *mock_stats_;
625 return SendStatisticsProxy::GetStats();
626 }
627
Niels Möller213618e2018-07-24 09:29:58 +0200628 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200629 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200630 if (mock_stats_)
631 return mock_stats_->input_frame_rate;
632 return SendStatisticsProxy::GetInputFrameRate();
633 }
sprangc5d62e22017-04-02 23:53:04 -0700634 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200635 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700636 mock_stats_.emplace(stats);
637 }
638
639 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200640 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700641 mock_stats_.reset();
642 }
643
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200644 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
645 on_frame_dropped_ = std::move(callback);
646 }
647
sprangc5d62e22017-04-02 23:53:04 -0700648 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200649 void OnFrameDropped(DropReason reason) override {
650 SendStatisticsProxy::OnFrameDropped(reason);
651 if (on_frame_dropped_)
652 on_frame_dropped_(reason);
653 }
654
Markus Handella3765182020-07-08 13:13:32 +0200655 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200656 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200657 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700658};
659
Markus Handellb4e96d42021-11-05 12:00:55 +0100660class SimpleVideoStreamEncoderFactory {
661 public:
662 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
663 public:
664 using VideoStreamEncoder::VideoStreamEncoder;
665 ~AdaptedVideoStreamEncoder() { Stop(); }
666 };
667
Markus Handell8d87c462021-12-16 11:37:16 +0100668 class MockFakeEncoder : public test::FakeEncoder {
669 public:
670 using FakeEncoder::FakeEncoder;
671 MOCK_METHOD(CodecSpecificInfo,
672 EncodeHook,
673 (EncodedImage & encoded_image,
674 rtc::scoped_refptr<EncodedImageBuffer> buffer),
675 (override));
676 };
677
Markus Handellee225432021-11-29 12:35:12 +0100678 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100679 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100680 encoder_settings_.bitrate_allocator_factory =
681 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100682 }
683
Markus Handell818e7fb2021-12-30 13:01:33 +0100684 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100685 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100686 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200687 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100688 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
689 time_controller_.GetClock(),
690 /*number_of_cores=*/1,
691 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
692 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100693 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100694 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100695 kVideoBitrateAllocation,
696 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100697 result->SetSink(&sink_, /*rotation_applied=*/false);
698 return result;
699 }
700
Markus Handell818e7fb2021-12-30 13:01:33 +0100701 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
702 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
703 TaskQueueBase** encoder_queue_ptr = nullptr) {
704 auto encoder_queue =
705 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
706 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
707 if (encoder_queue_ptr)
708 *encoder_queue_ptr = encoder_queue.get();
709 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
710 std::move(encoder_queue));
711 }
712
Markus Handell9a478b52021-11-18 16:07:01 +0100713 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100714 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100715
Markus Handell818e7fb2021-12-30 13:01:33 +0100716 GlobalSimulatedTimeController* GetTimeController() {
717 return &time_controller_;
718 }
719
Markus Handellb4e96d42021-11-05 12:00:55 +0100720 private:
721 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
722 public:
723 ~NullEncoderSink() override = default;
724 void OnEncoderConfigurationChanged(
725 std::vector<VideoStream> streams,
726 bool is_svc,
727 VideoEncoderConfig::ContentType content_type,
728 int min_transmit_bitrate_bps) override {}
729 void OnBitrateAllocationUpdated(
730 const VideoBitrateAllocation& allocation) override {}
731 void OnVideoLayersAllocationUpdated(
732 VideoLayersAllocation allocation) override {}
733 Result OnEncodedImage(
734 const EncodedImage& encoded_image,
735 const CodecSpecificInfo* codec_specific_info) override {
736 return Result(EncodedImageCallback::Result::OK);
737 }
738 };
739
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100740 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100741 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
742 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
743 time_controller_.CreateTaskQueueFactory()};
744 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
745 std::make_unique<MockableSendStatisticsProxy>(
746 time_controller_.GetClock(),
747 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100748 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
749 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100750 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
751 CreateBuiltinVideoBitrateAllocatorFactory();
752 VideoStreamEncoderSettings encoder_settings_{
753 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100754 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
755 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100756 NullEncoderSink sink_;
757};
758
759class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
760 public:
761 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100762 MOCK_METHOD(void,
763 SetZeroHertzModeEnabled,
764 (absl::optional<ZeroHertzModeParams>),
765 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100766 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100767 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
768 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100769 MOCK_METHOD(void,
770 UpdateLayerQualityConvergence,
771 (int spatial_index, bool converged),
772 (override));
773 MOCK_METHOD(void,
774 UpdateLayerStatus,
775 (int spatial_index, bool enabled),
776 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100777 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100778};
779
philipel9b058032020-02-10 11:30:00 +0100780class MockEncoderSelector
781 : public VideoEncoderFactory::EncoderSelectorInterface {
782 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200783 MOCK_METHOD(void,
784 OnCurrentEncoder,
785 (const SdpVideoFormat& format),
786 (override));
787 MOCK_METHOD(absl::optional<SdpVideoFormat>,
788 OnAvailableBitrate,
789 (const DataRate& rate),
790 (override));
791 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100792};
793
Markus Handell2e0f4f02021-12-21 19:14:58 +0100794class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
795 public:
796 MOCK_METHOD(void,
797 AddOrUpdateSink,
798 (rtc::VideoSinkInterface<VideoFrame>*,
799 const rtc::VideoSinkWants&),
800 (override));
801 MOCK_METHOD(void,
802 RemoveSink,
803 (rtc::VideoSinkInterface<VideoFrame>*),
804 (override));
805 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
806};
807
perkj803d97f2016-11-01 11:45:46 -0700808} // namespace
809
mflodmancc3d4422017-08-03 08:27:51 -0700810class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700811 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200812 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700813
mflodmancc3d4422017-08-03 08:27:51 -0700814 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700815 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700816 codec_width_(320),
817 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200818 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200819 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200820 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700821 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200822 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700823 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100824 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
825 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200826 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700827
828 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700829 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700830 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200831 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800832 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200833 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200834 video_send_config_.rtp.payload_name = "FAKE";
835 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700836
Per512ecb32016-09-23 15:52:06 +0200837 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200838 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200839 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
840 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
841 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100842 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700843
Niels Möllerf1338562018-04-26 09:51:47 +0200844 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800845 }
846
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100847 void ConfigureEncoder(
848 VideoEncoderConfig video_encoder_config,
849 VideoStreamEncoder::BitrateAllocationCallbackType
850 allocation_callback_type =
851 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200852 kVideoBitrateAllocationWhenScreenSharing,
853 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700854 if (video_stream_encoder_)
855 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100856
857 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
858 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
859 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
860 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
861 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100862 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100863 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
864 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
865 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200866 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200867 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700868 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700869 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200870 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700871 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200872 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800874 }
875
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100876 void ResetEncoder(const std::string& payload_name,
877 size_t num_streams,
878 size_t num_temporal_layers,
879 unsigned char num_spatial_layers,
880 bool screenshare,
881 VideoStreamEncoder::BitrateAllocationCallbackType
882 allocation_callback_type =
883 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200884 kVideoBitrateAllocationWhenScreenSharing,
885 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200886 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800887
888 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200889 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
890 num_streams, &video_encoder_config);
891 for (auto& layer : video_encoder_config.simulcast_layers) {
892 layer.num_temporal_layers = num_temporal_layers;
893 layer.max_framerate = kDefaultFramerate;
894 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100895 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200896 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700897 video_encoder_config.content_type =
898 screenshare ? VideoEncoderConfig::ContentType::kScreen
899 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700900 if (payload_name == "VP9") {
901 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
902 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200903 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700904 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200905 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
906 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700907 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200908 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
909 num_cores);
perkj26091b12016-09-01 01:17:40 -0700910 }
911
sprang57c2fff2017-01-16 06:24:02 -0800912 VideoFrame CreateFrame(int64_t ntp_time_ms,
913 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200914 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200915 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200916 destruction_event, codec_width_, codec_height_))
917 .set_ntp_time_ms(ntp_time_ms)
918 .set_timestamp_ms(99)
919 .set_rotation(kVideoRotation_0)
920 .build();
perkj26091b12016-09-01 01:17:40 -0700921 }
922
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100923 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
924 rtc::Event* destruction_event,
925 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200926 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200927 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200928 destruction_event, codec_width_, codec_height_))
929 .set_ntp_time_ms(ntp_time_ms)
930 .set_timestamp_ms(99)
931 .set_rotation(kVideoRotation_0)
932 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
933 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100934 }
935
sprang57c2fff2017-01-16 06:24:02 -0800936 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200937 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
938 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200939 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200940 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200941 .set_ntp_time_ms(ntp_time_ms)
942 .set_timestamp_ms(ntp_time_ms)
943 .set_rotation(kVideoRotation_0)
944 .build();
perkj803d97f2016-11-01 11:45:46 -0700945 }
946
Evan Shrubsole895556e2020-10-05 09:15:13 +0200947 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200948 return VideoFrame::Builder()
949 .set_video_frame_buffer(NV12Buffer::Create(width, height))
950 .set_ntp_time_ms(ntp_time_ms)
951 .set_timestamp_ms(ntp_time_ms)
952 .set_rotation(kVideoRotation_0)
953 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200954 }
955
Noah Richards51db4212019-06-12 06:59:12 -0700956 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
957 rtc::Event* destruction_event,
958 int width,
959 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200960 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200961 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200962 destruction_event, width, height))
963 .set_ntp_time_ms(ntp_time_ms)
964 .set_timestamp_ms(99)
965 .set_rotation(kVideoRotation_0)
966 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700967 }
968
Evan Shrubsole895556e2020-10-05 09:15:13 +0200969 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
970 rtc::Event* destruction_event,
971 int width,
972 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200973 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200974 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200975 destruction_event, width, height))
976 .set_ntp_time_ms(ntp_time_ms)
977 .set_timestamp_ms(99)
978 .set_rotation(kVideoRotation_0)
979 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200980 }
981
Noah Richards51db4212019-06-12 06:59:12 -0700982 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
983 rtc::Event* destruction_event) const {
984 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
985 codec_height_);
986 }
987
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100988 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200990 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100991
992 video_source_.IncomingCapturedFrame(
993 CreateFrame(1, codec_width_, codec_height_));
994 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200995 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100996 }
997
sprang4847ae62017-06-27 07:06:52 -0700998 void WaitForEncodedFrame(int64_t expected_ntp_time) {
999 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001000 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001001 }
1002
1003 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1004 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001005 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001006 return ok;
1007 }
1008
1009 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1010 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001011 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001012 }
1013
1014 void ExpectDroppedFrame() {
1015 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001016 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001017 }
1018
1019 bool WaitForFrame(int64_t timeout_ms) {
1020 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001021 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001022 return ok;
1023 }
1024
perkj26091b12016-09-01 01:17:40 -07001025 class TestEncoder : public test::FakeEncoder {
1026 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001027 explicit TestEncoder(TimeController* time_controller)
1028 : FakeEncoder(time_controller->GetClock()),
1029 time_controller_(time_controller) {
1030 RTC_DCHECK(time_controller_);
1031 }
perkj26091b12016-09-01 01:17:40 -07001032
Erik Språngaed30702018-11-05 12:57:17 +01001033 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001034 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001035 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001036 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001037 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001038 info.scaling_settings = VideoEncoder::ScalingSettings(
1039 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001040 }
1041 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001042 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1043 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001044 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001045 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001046 for (int tid = 0; tid < num_layers; ++tid)
1047 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001048 }
1049 }
Erik Språngaed30702018-11-05 12:57:17 +01001050 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001051
1052 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001053 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001054 info.apply_alignment_to_all_simulcast_layers =
1055 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001056 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001057 if (is_qp_trusted_.has_value()) {
1058 info.is_qp_trusted = is_qp_trusted_;
1059 }
Erik Språngaed30702018-11-05 12:57:17 +01001060 return info;
kthelgason876222f2016-11-29 01:44:11 -08001061 }
1062
Erik Språngb7cb7b52019-02-26 15:52:33 +01001063 int32_t RegisterEncodeCompleteCallback(
1064 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001065 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001066 encoded_image_callback_ = callback;
1067 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1068 }
1069
perkjfa10b552016-10-02 23:45:26 -07001070 void ContinueEncode() { continue_encode_event_.Set(); }
1071
1072 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1073 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001074 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001075 EXPECT_EQ(timestamp_, timestamp);
1076 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1077 }
1078
kthelgason2fc52542017-03-03 00:24:41 -08001079 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001080 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001081 quality_scaling_ = b;
1082 }
kthelgasonad9010c2017-02-14 00:46:51 -08001083
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001084 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001085 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001086 requested_resolution_alignment_ = requested_resolution_alignment;
1087 }
1088
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001089 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1090 MutexLock lock(&local_mutex_);
1091 apply_alignment_to_all_simulcast_layers_ = b;
1092 }
1093
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001094 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001095 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001096 is_hardware_accelerated_ = is_hardware_accelerated;
1097 }
1098
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001099 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1100 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001101 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001102 temporal_layers_supported_[spatial_idx] = supported;
1103 }
1104
Sergey Silkin6456e352019-07-08 17:56:40 +02001105 void SetResolutionBitrateLimits(
1106 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001107 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001108 resolution_bitrate_limits_ = thresholds;
1109 }
1110
sprangfe627f32017-03-29 08:24:59 -07001111 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001112 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001113 force_init_encode_failed_ = force_failure;
1114 }
1115
Niels Möller6bb5ab92019-01-11 11:11:10 +01001116 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001117 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001118 rate_factor_ = rate_factor;
1119 }
1120
Erik Språngd7329ca2019-02-21 21:19:53 +01001121 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001122 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001123 return last_framerate_;
1124 }
1125
Erik Språngd7329ca2019-02-21 21:19:53 +01001126 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001127 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001128 return last_update_rect_;
1129 }
1130
Niels Möller87e2d782019-03-07 10:18:23 +01001131 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001132 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 return last_frame_types_;
1134 }
1135
1136 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001137 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001138 keyframe ? VideoFrameType::kVideoFrameKey
1139 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001140 {
Markus Handella3765182020-07-08 13:13:32 +02001141 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001142 last_frame_types_ = frame_type;
1143 }
Niels Möllerb859b322019-03-07 12:40:01 +01001144 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001145 }
1146
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001147 void InjectEncodedImage(const EncodedImage& image,
1148 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001149 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001150 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001151 }
1152
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001153 void SetEncodedImageData(
1154 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001155 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001156 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001157 }
1158
Erik Språngd7329ca2019-02-21 21:19:53 +01001159 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001160 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001161 expect_null_frame_ = true;
1162 }
1163
Erik Språng5056af02019-09-02 15:53:11 +02001164 absl::optional<VideoEncoder::RateControlParameters>
1165 GetAndResetLastRateControlSettings() {
1166 auto settings = last_rate_control_settings_;
1167 last_rate_control_settings_.reset();
1168 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001169 }
1170
Henrik Boström56db9ff2021-03-24 09:06:45 +01001171 int GetLastInputWidth() const {
1172 MutexLock lock(&local_mutex_);
1173 return last_input_width_;
1174 }
1175
1176 int GetLastInputHeight() const {
1177 MutexLock lock(&local_mutex_);
1178 return last_input_height_;
1179 }
1180
Evan Shrubsole895556e2020-10-05 09:15:13 +02001181 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1182 MutexLock lock(&local_mutex_);
1183 return last_input_pixel_format_;
1184 }
1185
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001186 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001187 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001188 return num_set_rates_;
1189 }
1190
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001191 void SetPreferredPixelFormats(
1192 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1193 pixel_formats) {
1194 MutexLock lock(&local_mutex_);
1195 preferred_pixel_formats_ = std::move(pixel_formats);
1196 }
1197
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001198 void SetIsQpTrusted(absl::optional<bool> trusted) {
1199 MutexLock lock(&local_mutex_);
1200 is_qp_trusted_ = trusted;
1201 }
1202
Erik Språnge4589cb2022-04-06 16:44:30 +02001203 VideoCodecComplexity LastEncoderComplexity() {
1204 MutexLock lock(&local_mutex_);
1205 return last_encoder_complexity_;
1206 }
1207
perkjfa10b552016-10-02 23:45:26 -07001208 private:
perkj26091b12016-09-01 01:17:40 -07001209 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001210 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001211 {
Markus Handella3765182020-07-08 13:13:32 +02001212 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001213 if (expect_null_frame_) {
1214 EXPECT_EQ(input_image.timestamp(), 0u);
1215 EXPECT_EQ(input_image.width(), 1);
1216 last_frame_types_ = *frame_types;
1217 expect_null_frame_ = false;
1218 } else {
1219 EXPECT_GT(input_image.timestamp(), timestamp_);
1220 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1221 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1222 }
perkj26091b12016-09-01 01:17:40 -07001223
1224 timestamp_ = input_image.timestamp();
1225 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001226 last_input_width_ = input_image.width();
1227 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001228 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001229 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001230 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001231 }
Niels Möllerb859b322019-03-07 12:40:01 +01001232 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001233 return result;
1234 }
1235
Niels Möller08ae7ce2020-09-23 15:58:12 +02001236 CodecSpecificInfo EncodeHook(
1237 EncodedImage& encoded_image,
1238 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001239 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001240 {
1241 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001242 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001243 }
1244 MutexLock lock(&local_mutex_);
1245 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001246 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001247 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001248 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001249 }
1250
sprangfe627f32017-03-29 08:24:59 -07001251 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001252 const Settings& settings) override {
1253 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001254
Markus Handella3765182020-07-08 13:13:32 +02001255 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001256 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001257
Erik Språng82fad3d2018-03-21 09:57:23 +01001258 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001259 // Simulate setting up temporal layers, in order to validate the life
1260 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001261 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001262 frame_buffer_controller_ =
1263 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001264 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001265
1266 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1267
Erik Språngb7cb7b52019-02-26 15:52:33 +01001268 if (force_init_encode_failed_) {
1269 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001270 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001271 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001272
Erik Språngb7cb7b52019-02-26 15:52:33 +01001273 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001274 return res;
1275 }
1276
Erik Språngb7cb7b52019-02-26 15:52:33 +01001277 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001278 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001279 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1280 initialized_ = EncoderState::kUninitialized;
1281 return FakeEncoder::Release();
1282 }
1283
Erik Språng16cb8f52019-04-12 13:59:09 +02001284 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001285 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001286 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001287 VideoBitrateAllocation adjusted_rate_allocation;
1288 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1289 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001290 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001291 adjusted_rate_allocation.SetBitrate(
1292 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001293 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001294 rate_factor_));
1295 }
1296 }
1297 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001298 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001299 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001300 RateControlParameters adjusted_paramters = parameters;
1301 adjusted_paramters.bitrate = adjusted_rate_allocation;
1302 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001303 }
1304
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001305 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001306 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001307 enum class EncoderState {
1308 kUninitialized,
1309 kInitializationFailed,
1310 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001311 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001312 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001313 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1314 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1315 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1316 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1317 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1318 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001319 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1320 false;
Markus Handella3765182020-07-08 13:13:32 +02001321 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001322 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1323 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001324 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001325 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001326 absl::optional<bool>
1327 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001328 local_mutex_);
1329 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1330 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1331 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001332 absl::optional<VideoEncoder::RateControlParameters>
1333 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001334 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1335 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001336 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001337 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001338 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1339 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001340 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001341 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001342 RTC_GUARDED_BY(local_mutex_);
1343 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001344 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1345 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001346 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1347 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001348 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001349 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1350 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001351 };
1352
mflodmancc3d4422017-08-03 08:27:51 -07001353 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001354 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001355 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1356 : time_controller_(time_controller), test_encoder_(test_encoder) {
1357 RTC_DCHECK(time_controller_);
1358 }
perkj26091b12016-09-01 01:17:40 -07001359
perkj26091b12016-09-01 01:17:40 -07001360 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001361 EXPECT_TRUE(
1362 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1363 }
1364
1365 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1366 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001367 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001368 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001369 return false;
perkj26091b12016-09-01 01:17:40 -07001370 {
Markus Handella3765182020-07-08 13:13:32 +02001371 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001372 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001373 }
1374 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001375 return true;
perkj26091b12016-09-01 01:17:40 -07001376 }
1377
sprangb1ca0732017-02-01 08:38:12 -08001378 void WaitForEncodedFrame(uint32_t expected_width,
1379 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001380 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001381 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001382 }
1383
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001384 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001385 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001386 uint32_t width = 0;
1387 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001388 {
Markus Handella3765182020-07-08 13:13:32 +02001389 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001390 width = last_width_;
1391 height = last_height_;
1392 }
1393 EXPECT_EQ(expected_height, height);
1394 EXPECT_EQ(expected_width, width);
1395 }
1396
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001397 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1398 VideoRotation rotation;
1399 {
Markus Handella3765182020-07-08 13:13:32 +02001400 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001401 rotation = last_rotation_;
1402 }
1403 EXPECT_EQ(expected_rotation, rotation);
1404 }
1405
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001406 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001407
sprangc5d62e22017-04-02 23:53:04 -07001408 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001409 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001410 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001411 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001412 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001413 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001414 }
1415
perkj26091b12016-09-01 01:17:40 -07001416 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001417 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001418 expect_frames_ = false;
1419 }
1420
asaperssonfab67072017-04-04 05:51:49 -07001421 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001422 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001423 return number_of_reconfigurations_;
1424 }
1425
asaperssonfab67072017-04-04 05:51:49 -07001426 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001427 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001428 return min_transmit_bitrate_bps_;
1429 }
1430
Erik Språngd7329ca2019-02-21 21:19:53 +01001431 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001432 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001433 num_expected_layers_ = num_layers;
1434 }
1435
Erik Språngb7cb7b52019-02-26 15:52:33 +01001436 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001437 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001438 return last_capture_time_ms_;
1439 }
1440
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001441 const EncodedImage& GetLastEncodedImage() {
1442 MutexLock lock(&mutex_);
1443 return last_encoded_image_;
1444 }
1445
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001446 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001447 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001448 return std::move(last_encoded_image_data_);
1449 }
1450
Per Kjellanderdcef6412020-10-07 15:09:05 +02001451 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1452 MutexLock lock(&mutex_);
1453 return last_bitrate_allocation_;
1454 }
1455
1456 int number_of_bitrate_allocations() const {
1457 MutexLock lock(&mutex_);
1458 return number_of_bitrate_allocations_;
1459 }
1460
Per Kjellandera9434842020-10-15 17:53:22 +02001461 VideoLayersAllocation GetLastVideoLayersAllocation() {
1462 MutexLock lock(&mutex_);
1463 return last_layers_allocation_;
1464 }
1465
1466 int number_of_layers_allocations() const {
1467 MutexLock lock(&mutex_);
1468 return number_of_layers_allocations_;
1469 }
1470
perkj26091b12016-09-01 01:17:40 -07001471 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001472 Result OnEncodedImage(
1473 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001474 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001475 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001476 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001477 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001478 last_encoded_image_data_ = std::vector<uint8_t>(
1479 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001480 uint32_t timestamp = encoded_image.Timestamp();
1481 if (last_timestamp_ != timestamp) {
1482 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001483 last_width_ = encoded_image._encodedWidth;
1484 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001485 } else {
1486 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001487 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1488 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001489 }
1490 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001491 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001492 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001493 if (num_received_layers_ == num_expected_layers_) {
1494 encoded_frame_event_.Set();
1495 }
sprangb1ca0732017-02-01 08:38:12 -08001496 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001497 }
1498
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001499 void OnEncoderConfigurationChanged(
1500 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001501 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001502 VideoEncoderConfig::ContentType content_type,
1503 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001504 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001505 ++number_of_reconfigurations_;
1506 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1507 }
1508
Per Kjellanderdcef6412020-10-07 15:09:05 +02001509 void OnBitrateAllocationUpdated(
1510 const VideoBitrateAllocation& allocation) override {
1511 MutexLock lock(&mutex_);
1512 ++number_of_bitrate_allocations_;
1513 last_bitrate_allocation_ = allocation;
1514 }
1515
Per Kjellandera9434842020-10-15 17:53:22 +02001516 void OnVideoLayersAllocationUpdated(
1517 VideoLayersAllocation allocation) override {
1518 MutexLock lock(&mutex_);
1519 ++number_of_layers_allocations_;
1520 last_layers_allocation_ = allocation;
1521 rtc::StringBuilder log;
1522 for (const auto& layer : allocation.active_spatial_layers) {
1523 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1524 << "[";
1525 for (const auto target_bitrate :
1526 layer.target_bitrate_per_temporal_layer) {
1527 log << target_bitrate.kbps() << ",";
1528 }
1529 log << "]";
1530 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001531 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001532 }
1533
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001534 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001535 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001536 TestEncoder* test_encoder_;
1537 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001538 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001539 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001540 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001541 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001542 uint32_t last_height_ = 0;
1543 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001544 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001545 size_t num_expected_layers_ = 1;
1546 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001547 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001548 int number_of_reconfigurations_ = 0;
1549 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001550 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1551 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001552 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1553 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001554 };
1555
Sergey Silkin5ee69672019-07-02 14:18:34 +02001556 class VideoBitrateAllocatorProxyFactory
1557 : public VideoBitrateAllocatorFactory {
1558 public:
1559 VideoBitrateAllocatorProxyFactory()
1560 : bitrate_allocator_factory_(
1561 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1562
1563 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1564 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001565 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001566 codec_config_ = codec;
1567 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1568 }
1569
1570 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001571 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001572 return codec_config_;
1573 }
1574
1575 private:
1576 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1577
Markus Handella3765182020-07-08 13:13:32 +02001578 mutable Mutex mutex_;
1579 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001580 };
1581
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001582 Clock* clock() { return time_controller_.GetClock(); }
1583 void AdvanceTime(TimeDelta duration) {
1584 time_controller_.AdvanceTime(duration);
1585 }
1586
1587 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1588
1589 protected:
1590 virtual TaskQueueFactory* GetTaskQueueFactory() {
1591 return time_controller_.GetTaskQueueFactory();
1592 }
1593
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001594 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001595 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001596 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001597 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001598 int codec_width_;
1599 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001600 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001601 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001602 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001603 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001604 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001605 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001606 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001607 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001608};
1609
mflodmancc3d4422017-08-03 08:27:51 -07001610TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001612 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001613 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001614 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001615 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001616 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001617 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001618}
1619
mflodmancc3d4422017-08-03 08:27:51 -07001620TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001621 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001622 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001623 // The encoder will cache up to one frame for a short duration. Adding two
1624 // frames means that the first frame will be dropped and the second frame will
1625 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001626 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001627 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001628 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001629 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001630 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001631
Henrik Boström381d1092020-05-12 18:49:07 +02001632 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001633 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001634
Sebastian Janssona3177052018-04-10 13:05:49 +02001635 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001636 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001637 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1638
1639 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001640 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001641}
1642
mflodmancc3d4422017-08-03 08:27:51 -07001643TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001646 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001647 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001648
Henrik Boström381d1092020-05-12 18:49:07 +02001649 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001650 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1651
Sebastian Janssona3177052018-04-10 13:05:49 +02001652 // The encoder will cache up to one frame for a short duration. Adding two
1653 // frames means that the first frame will be dropped and the second frame will
1654 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001655 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001656 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001657
Henrik Boström381d1092020-05-12 18:49:07 +02001658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001659 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001660 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001661 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1662 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001663 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001664}
1665
mflodmancc3d4422017-08-03 08:27:51 -07001666TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001668 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001669 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001670 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001671
1672 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001673 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001674
perkja49cbd32016-09-16 07:53:41 -07001675 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001676 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001677 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001678}
1679
mflodmancc3d4422017-08-03 08:27:51 -07001680TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001682 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001683
perkja49cbd32016-09-16 07:53:41 -07001684 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001685 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001686
mflodmancc3d4422017-08-03 08:27:51 -07001687 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001688 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001689 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001690 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1691 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001692}
1693
Markus Handell9a478b52021-11-18 16:07:01 +01001694TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1695 test::FrameForwarder source;
1696 video_stream_encoder_->SetSource(&source,
1697 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001700
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001701 int dropped_count = 0;
1702 stats_proxy_->SetDroppedFrameCallback(
1703 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1704 ++dropped_count;
1705 });
1706
Markus Handell9a478b52021-11-18 16:07:01 +01001707 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1708 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1709 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001710 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001711 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001712}
1713
Henrik Boström56db9ff2021-03-24 09:06:45 +01001714TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001716 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001717
1718 rtc::Event frame_destroyed_event;
1719 video_source_.IncomingCapturedFrame(
1720 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001721 WaitForEncodedFrame(1);
1722 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1723 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001724 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1725 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001726 video_stream_encoder_->Stop();
1727}
1728
Henrik Boström56db9ff2021-03-24 09:06:45 +01001729TEST_F(VideoStreamEncoderTest,
1730 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001731 // Use the cropping factory.
1732 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001733 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001734 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1735 kMaxPayloadLength);
1736 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1737
1738 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001740 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001741 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1742 WaitForEncodedFrame(1);
1743 // The encoder will have been configured once.
1744 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001745 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1746 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001747
1748 // Now send in a fake frame that needs to be cropped as the width/height
1749 // aren't divisible by 4 (see CreateEncoderStreams above).
1750 rtc::Event frame_destroyed_event;
1751 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1752 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001753 WaitForEncodedFrame(2);
1754 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1755 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001756 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1757 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001758 video_stream_encoder_->Stop();
1759}
1760
Evan Shrubsole895556e2020-10-05 09:15:13 +02001761TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1762 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001763 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001764
1765 video_source_.IncomingCapturedFrame(
1766 CreateNV12Frame(1, codec_width_, codec_height_));
1767 WaitForEncodedFrame(1);
1768 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1769 fake_encoder_.GetLastInputPixelFormat());
1770 video_stream_encoder_->Stop();
1771}
1772
Henrik Boström56db9ff2021-03-24 09:06:45 +01001773TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001775 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001776
1777 fake_encoder_.SetPreferredPixelFormats({});
1778
1779 rtc::Event frame_destroyed_event;
1780 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1781 1, &frame_destroyed_event, codec_width_, codec_height_));
1782 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001783 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001784 fake_encoder_.GetLastInputPixelFormat());
1785 video_stream_encoder_->Stop();
1786}
1787
Henrik Boström56db9ff2021-03-24 09:06:45 +01001788TEST_F(VideoStreamEncoderTest,
1789 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001791 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001792
1793 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1794
1795 rtc::Event frame_destroyed_event;
1796 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1797 1, &frame_destroyed_event, codec_width_, codec_height_));
1798 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001799 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001800 fake_encoder_.GetLastInputPixelFormat());
1801 video_stream_encoder_->Stop();
1802}
1803
Henrik Boström56db9ff2021-03-24 09:06:45 +01001804TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001805 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001806 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001807
1808 // Fake NV12 native frame does not allow mapping to I444.
1809 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1810
1811 rtc::Event frame_destroyed_event;
1812 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1813 1, &frame_destroyed_event, codec_width_, codec_height_));
1814 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001815 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001816 fake_encoder_.GetLastInputPixelFormat());
1817 video_stream_encoder_->Stop();
1818}
1819
Henrik Boström56db9ff2021-03-24 09:06:45 +01001820TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001822 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001823
1824 rtc::Event frame_destroyed_event;
1825 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1826 1, &frame_destroyed_event, codec_width_, codec_height_));
1827 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001828 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001829 fake_encoder_.GetLastInputPixelFormat());
1830 video_stream_encoder_->Stop();
1831}
1832
Ying Wang9b881ab2020-02-07 14:29:32 +01001833TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001835 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001836 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1837 WaitForEncodedFrame(1);
1838
Henrik Boström381d1092020-05-12 18:49:07 +02001839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001840 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001841 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1842 // frames. Adding two frames means that the first frame will be dropped and
1843 // the second frame will be sent to the encoder.
1844 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1845 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1846 WaitForEncodedFrame(3);
1847 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1848 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1849 WaitForEncodedFrame(5);
1850 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1851 video_stream_encoder_->Stop();
1852}
1853
mflodmancc3d4422017-08-03 08:27:51 -07001854TEST_F(VideoStreamEncoderTest,
1855 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001857 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001858 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001859
1860 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001861 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001862 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001863 // The encoder will have been configured once when the first frame is
1864 // received.
1865 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001866
1867 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001868 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001869 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001870 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001871 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001872
1873 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001874 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001875 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001876 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001877 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001878
mflodmancc3d4422017-08-03 08:27:51 -07001879 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001880}
1881
mflodmancc3d4422017-08-03 08:27:51 -07001882TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001883 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001884 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001885
1886 // Capture a frame and wait for it to synchronize with the encoder thread.
1887 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001888 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001889 // The encoder will have been configured once.
1890 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001891 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1892 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001893
1894 codec_width_ *= 2;
1895 codec_height_ *= 2;
1896 // Capture a frame with a higher resolution and wait for it to synchronize
1897 // with the encoder thread.
1898 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001899 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001900 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1901 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001902 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001903
mflodmancc3d4422017-08-03 08:27:51 -07001904 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001905}
1906
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001907TEST_F(VideoStreamEncoderTest,
1908 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001910 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001911
1912 // Capture a frame and wait for it to synchronize with the encoder thread.
1913 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1914 WaitForEncodedFrame(1);
1915
1916 VideoEncoderConfig video_encoder_config;
1917 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1918 // Changing the max payload data length recreates encoder.
1919 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1920 kMaxPayloadLength / 2);
1921
1922 // Capture a frame and wait for it to synchronize with the encoder thread.
1923 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1924 WaitForEncodedFrame(2);
1925 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1926
1927 video_stream_encoder_->Stop();
1928}
1929
Sergey Silkin5ee69672019-07-02 14:18:34 +02001930TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001933
1934 VideoEncoderConfig video_encoder_config;
1935 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001936 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1937 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001938 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1939 kMaxPayloadLength);
1940
1941 // Capture a frame and wait for it to synchronize with the encoder thread.
1942 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1943 WaitForEncodedFrame(1);
1944 // The encoder will have been configured once when the first frame is
1945 // received.
1946 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001947 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001948 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001949 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001950 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1951
Sergey Silkin6456e352019-07-08 17:56:40 +02001952 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1953 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001954 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1955 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001956 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1957 kMaxPayloadLength);
1958
1959 // Capture a frame and wait for it to synchronize with the encoder thread.
1960 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1961 WaitForEncodedFrame(2);
1962 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1963 // Bitrate limits have changed - rate allocator should be reconfigured,
1964 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001965 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001966 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001967 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001968 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001969 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001970
1971 video_stream_encoder_->Stop();
1972}
1973
Sergey Silkin6456e352019-07-08 17:56:40 +02001974TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001975 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001977 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001978
Sergey Silkincd02eba2020-01-20 14:48:40 +01001979 const uint32_t kMinEncBitrateKbps = 100;
1980 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001981 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001982 /*frame_size_pixels=*/codec_width_ * codec_height_,
1983 /*min_start_bitrate_bps=*/0,
1984 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1985 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001986 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1987
Sergey Silkincd02eba2020-01-20 14:48:40 +01001988 VideoEncoderConfig video_encoder_config;
1989 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1990 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1991 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1992 (kMinEncBitrateKbps + 1) * 1000;
1993 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1994 kMaxPayloadLength);
1995
1996 // When both encoder and app provide bitrate limits, the intersection of
1997 // provided sets should be used.
1998 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1999 WaitForEncodedFrame(1);
2000 EXPECT_EQ(kMaxEncBitrateKbps,
2001 bitrate_allocator_factory_.codec_config().maxBitrate);
2002 EXPECT_EQ(kMinEncBitrateKbps + 1,
2003 bitrate_allocator_factory_.codec_config().minBitrate);
2004
2005 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2006 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2007 (kMinEncBitrateKbps - 1) * 1000;
2008 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2009 kMaxPayloadLength);
2010 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002011 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002012 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002013 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002014 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002015 bitrate_allocator_factory_.codec_config().minBitrate);
2016
Sergey Silkincd02eba2020-01-20 14:48:40 +01002017 video_stream_encoder_->Stop();
2018}
2019
2020TEST_F(VideoStreamEncoderTest,
2021 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002023 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002024
2025 const uint32_t kMinAppBitrateKbps = 100;
2026 const uint32_t kMaxAppBitrateKbps = 200;
2027 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2028 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2029 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2030 /*frame_size_pixels=*/codec_width_ * codec_height_,
2031 /*min_start_bitrate_bps=*/0,
2032 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2033 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2034 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2035
2036 VideoEncoderConfig video_encoder_config;
2037 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2038 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2039 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2040 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002041 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2042 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002043
Sergey Silkincd02eba2020-01-20 14:48:40 +01002044 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2045 WaitForEncodedFrame(1);
2046 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002047 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002048 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002049 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002050
2051 video_stream_encoder_->Stop();
2052}
2053
2054TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002055 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002057 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002058
2059 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002060 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002061 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002062 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002063 fake_encoder_.SetResolutionBitrateLimits(
2064 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2065
2066 VideoEncoderConfig video_encoder_config;
2067 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2068 video_encoder_config.max_bitrate_bps = 0;
2069 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2070 kMaxPayloadLength);
2071
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002072 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002073 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2074 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002075 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2076 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002077 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2078 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2079
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002080 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002081 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2082 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002083 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2084 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002085 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2086 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2087
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002088 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002089 // encoder for 360p should be used.
2090 video_source_.IncomingCapturedFrame(
2091 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2092 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002093 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2094 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002095 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2096 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2097
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002098 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002099 // ignored.
2100 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2101 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002102 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2103 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002104 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2105 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002106 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2107 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002108 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2109 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2110
2111 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2112 // for 270p should be used.
2113 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2114 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002115 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2116 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002117 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2118 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2119
2120 video_stream_encoder_->Stop();
2121}
2122
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002123TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002125 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002126
2127 VideoEncoderConfig video_encoder_config;
2128 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2129 video_encoder_config.max_bitrate_bps = 0;
2130 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2131 kMaxPayloadLength);
2132
2133 // Encode 720p frame to get the default encoder target bitrate.
2134 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2135 WaitForEncodedFrame(1);
2136 const uint32_t kDefaultTargetBitrateFor720pKbps =
2137 bitrate_allocator_factory_.codec_config()
2138 .simulcastStream[0]
2139 .targetBitrate;
2140
2141 // Set the max recommended encoder bitrate to something lower than the default
2142 // target bitrate.
2143 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2144 1280 * 720, 10 * 1000, 10 * 1000,
2145 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2146 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2147
2148 // Change resolution to trigger encoder reinitialization.
2149 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2150 WaitForEncodedFrame(2);
2151 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2152 WaitForEncodedFrame(3);
2153
2154 // Ensure the target bitrate is capped by the max bitrate.
2155 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2156 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2157 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2158 .simulcastStream[0]
2159 .targetBitrate *
2160 1000,
2161 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2162
2163 video_stream_encoder_->Stop();
2164}
2165
Åsa Perssona7e34d32021-01-20 15:36:13 +01002166TEST_F(VideoStreamEncoderTest,
2167 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2168 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2169 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2170 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2171 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2172 fake_encoder_.SetResolutionBitrateLimits(
2173 {kEncoderLimits270p, kEncoderLimits360p});
2174
2175 // Two streams, highest stream active.
2176 VideoEncoderConfig config;
2177 const int kNumStreams = 2;
2178 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2179 config.max_bitrate_bps = 0;
2180 config.simulcast_layers[0].active = false;
2181 config.simulcast_layers[1].active = true;
2182 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002183 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002184 "VP8", /*max qp*/ 56, /*screencast*/ false,
2185 /*screenshare enabled*/ false);
2186 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2187
2188 // The encoder bitrate limits for 270p should be used.
2189 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2190 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002191 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002192 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002193 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002194 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002195 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002196
2197 // The encoder bitrate limits for 360p should be used.
2198 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2199 EXPECT_FALSE(WaitForFrame(1000));
2200 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002201 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002202 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204
2205 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2206 video_source_.IncomingCapturedFrame(
2207 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2208 EXPECT_FALSE(WaitForFrame(1000));
2209 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002210 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002211 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002212 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002213
2214 // Resolution higher than 360p. Encoder limits should be ignored.
2215 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2216 EXPECT_FALSE(WaitForFrame(1000));
2217 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002218 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002219 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002220 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002221 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225
2226 // Resolution lower than 270p. The encoder limits for 270p should be used.
2227 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2228 EXPECT_FALSE(WaitForFrame(1000));
2229 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233
2234 video_stream_encoder_->Stop();
2235}
2236
2237TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002238 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2239 // Two streams, highest stream active.
2240 VideoEncoderConfig config;
2241 const int kNumStreams = 2;
2242 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2243 config.max_bitrate_bps = 0;
2244 config.simulcast_layers[0].active = false;
2245 config.simulcast_layers[1].active = true;
2246 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002247 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002248 "VP8", /*max qp*/ 56, /*screencast*/ false,
2249 /*screenshare enabled*/ false);
2250 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2251
2252 // Default bitrate limits for 270p should be used.
2253 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2254 kDefaultLimits270p =
2255 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002256 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002257 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2258 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002259 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002260 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002261 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002262 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002263 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002264
2265 // Default bitrate limits for 360p should be used.
2266 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2267 kDefaultLimits360p =
2268 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002269 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002270 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2271 EXPECT_FALSE(WaitForFrame(1000));
2272 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002273 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002274 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002276
2277 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2278 video_source_.IncomingCapturedFrame(
2279 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2280 EXPECT_FALSE(WaitForFrame(1000));
2281 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002282 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002283 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002284 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002285
2286 // Default bitrate limits for 540p should be used.
2287 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2288 kDefaultLimits540p =
2289 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002290 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002291 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2292 EXPECT_FALSE(WaitForFrame(1000));
2293 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002294 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002295 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002296 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002297
2298 video_stream_encoder_->Stop();
2299}
2300
2301TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002302 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2303 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2304 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2305 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2306 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2307 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2308 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2309 fake_encoder_.SetResolutionBitrateLimits(
2310 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2311
2312 // Three streams, middle stream active.
2313 VideoEncoderConfig config;
2314 const int kNumStreams = 3;
2315 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2316 config.simulcast_layers[0].active = false;
2317 config.simulcast_layers[1].active = true;
2318 config.simulcast_layers[2].active = false;
2319 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002320 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002321 "VP8", /*max qp*/ 56, /*screencast*/ false,
2322 /*screenshare enabled*/ false);
2323 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2324
2325 // The encoder bitrate limits for 360p should be used.
2326 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2327 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002328 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002329 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002330 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002331 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002332 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002333
2334 // The encoder bitrate limits for 270p should be used.
2335 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2336 EXPECT_FALSE(WaitForFrame(1000));
2337 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002338 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002339 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002340 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341
2342 video_stream_encoder_->Stop();
2343}
2344
2345TEST_F(VideoStreamEncoderTest,
2346 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2347 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2348 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2349 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2350 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2351 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2352 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2353 fake_encoder_.SetResolutionBitrateLimits(
2354 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2355
2356 // Three streams, lowest stream active.
2357 VideoEncoderConfig config;
2358 const int kNumStreams = 3;
2359 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2360 config.simulcast_layers[0].active = true;
2361 config.simulcast_layers[1].active = false;
2362 config.simulcast_layers[2].active = false;
2363 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002364 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002365 "VP8", /*max qp*/ 56, /*screencast*/ false,
2366 /*screenshare enabled*/ false);
2367 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2368
2369 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2370 // on lowest stream, limits for 270p should not be used
2371 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2372 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002373 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002374 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002375 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002376 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002377 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002378
2379 video_stream_encoder_->Stop();
2380}
2381
2382TEST_F(VideoStreamEncoderTest,
2383 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2384 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2385 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2386 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2387 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2388 fake_encoder_.SetResolutionBitrateLimits(
2389 {kEncoderLimits270p, kEncoderLimits360p});
2390 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2391
2392 // Two streams, highest stream active.
2393 VideoEncoderConfig config;
2394 const int kNumStreams = 2;
2395 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2396 config.simulcast_layers[0].active = false;
2397 config.simulcast_layers[1].active = true;
2398 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2399 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002400 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002401 "VP8", /*max qp*/ 56, /*screencast*/ false,
2402 /*screenshare enabled*/ false);
2403 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2404
2405 // The encoder bitrate limits for 270p should be used.
2406 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2407 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002408 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002409 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002410 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002411 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002412 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002413
2414 // The max configured bitrate is less than the encoder limit for 360p.
2415 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2416 EXPECT_FALSE(WaitForFrame(1000));
2417 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002418 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002419 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002420 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002421
2422 video_stream_encoder_->Stop();
2423}
2424
mflodmancc3d4422017-08-03 08:27:51 -07002425TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002426 EXPECT_TRUE(video_source_.has_sinks());
2427 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002428 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002429 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002430 EXPECT_FALSE(video_source_.has_sinks());
2431 EXPECT_TRUE(new_video_source.has_sinks());
2432
mflodmancc3d4422017-08-03 08:27:51 -07002433 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002434}
2435
mflodmancc3d4422017-08-03 08:27:51 -07002436TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002437 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002438 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002439 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002441}
2442
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002443class ResolutionAlignmentTest
2444 : public VideoStreamEncoderTest,
2445 public ::testing::WithParamInterface<
2446 ::testing::tuple<int, std::vector<double>>> {
2447 public:
2448 ResolutionAlignmentTest()
2449 : requested_alignment_(::testing::get<0>(GetParam())),
2450 scale_factors_(::testing::get<1>(GetParam())) {}
2451
2452 protected:
2453 const int requested_alignment_;
2454 const std::vector<double> scale_factors_;
2455};
2456
2457INSTANTIATE_TEST_SUITE_P(
2458 AlignmentAndScaleFactors,
2459 ResolutionAlignmentTest,
2460 ::testing::Combine(
2461 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2462 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2463 std::vector<double>{-1.0, -1.0},
2464 std::vector<double>{-1.0, -1.0, -1.0},
2465 std::vector<double>{4.0, 2.0, 1.0},
2466 std::vector<double>{9999.0, -1.0, 1.0},
2467 std::vector<double>{3.99, 2.01, 1.0},
2468 std::vector<double>{4.9, 1.7, 1.25},
2469 std::vector<double>{10.0, 4.0, 3.0},
2470 std::vector<double>{1.75, 3.5},
2471 std::vector<double>{1.5, 2.5},
2472 std::vector<double>{1.3, 1.0})));
2473
2474TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2475 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002476 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002477 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2478 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2479
2480 // Fill config with the scaling factor by which to reduce encoding size.
2481 const int num_streams = scale_factors_.size();
2482 VideoEncoderConfig config;
2483 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2484 for (int i = 0; i < num_streams; ++i) {
2485 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2486 }
2487 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002488 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002489 "VP8", /*max qp*/ 56, /*screencast*/ false,
2490 /*screenshare enabled*/ false);
2491 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2492
Henrik Boström381d1092020-05-12 18:49:07 +02002493 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002494 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2495 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002496 // Wait for all layers before triggering event.
2497 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002498
2499 // On the 1st frame, we should have initialized the encoder and
2500 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002501 int64_t timestamp_ms = kFrameIntervalMs;
2502 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2503 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002504 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002505
2506 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2507 // (It's up the to the encoder to potentially drop the previous frame,
2508 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002509 timestamp_ms += kFrameIntervalMs;
2510 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2511 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002512 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002513
Asa Persson606d3cb2021-10-04 10:07:11 +02002514 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002515 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2516 // Frame size should be a multiple of the requested alignment.
2517 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2518 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2519 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2520 // Aspect ratio should match.
2521 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2522 codec.height * codec.simulcastStream[i].width);
2523 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002524
2525 video_stream_encoder_->Stop();
2526}
2527
Jonathan Yubc771b72017-12-08 17:04:29 -08002528TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2529 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002530 const int kWidth = 1280;
2531 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002532
2533 // We rely on the automatic resolution adaptation, but we handle framerate
2534 // adaptation manually by mocking the stats proxy.
2535 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002536
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002537 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002539 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002540 video_stream_encoder_->SetSource(&video_source_,
2541 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002542 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002543 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002544 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002545 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2546
Jonathan Yubc771b72017-12-08 17:04:29 -08002547 // Adapt down as far as possible.
2548 rtc::VideoSinkWants last_wants;
2549 int64_t t = 1;
2550 int loop_count = 0;
2551 do {
2552 ++loop_count;
2553 last_wants = video_source_.sink_wants();
2554
2555 // Simulate the framerate we've been asked to adapt to.
2556 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2557 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2558 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2559 mock_stats.input_frame_rate = fps;
2560 stats_proxy_->SetMockStats(mock_stats);
2561
2562 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2563 sink_.WaitForEncodedFrame(t);
2564 t += frame_interval_ms;
2565
mflodmancc3d4422017-08-03 08:27:51 -07002566 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002567 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002568 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002569 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2570 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002571 } while (video_source_.sink_wants().max_pixel_count <
2572 last_wants.max_pixel_count ||
2573 video_source_.sink_wants().max_framerate_fps <
2574 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002575
Jonathan Yubc771b72017-12-08 17:04:29 -08002576 // Verify that we've adapted all the way down.
2577 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002578 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002579 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2580 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002581 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002582 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2583 *video_source_.last_sent_height());
2584 EXPECT_EQ(kMinBalancedFramerateFps,
2585 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002586
Jonathan Yubc771b72017-12-08 17:04:29 -08002587 // Adapt back up the same number of times we adapted down.
2588 for (int i = 0; i < loop_count - 1; ++i) {
2589 last_wants = video_source_.sink_wants();
2590
2591 // Simulate the framerate we've been asked to adapt to.
2592 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2593 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2594 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2595 mock_stats.input_frame_rate = fps;
2596 stats_proxy_->SetMockStats(mock_stats);
2597
2598 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2599 sink_.WaitForEncodedFrame(t);
2600 t += frame_interval_ms;
2601
Henrik Boström91aa7322020-04-28 12:24:33 +02002602 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002603 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002604 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002605 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2606 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002607 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2608 last_wants.max_pixel_count ||
2609 video_source_.sink_wants().max_framerate_fps >
2610 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002611 }
2612
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002613 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002614 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002616 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2617 EXPECT_EQ((loop_count - 1) * 2,
2618 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002619
mflodmancc3d4422017-08-03 08:27:51 -07002620 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002621}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002622
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002623TEST_F(VideoStreamEncoderTest,
2624 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002625 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2626 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002627 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002628
2629 const int kFrameWidth = 1280;
2630 const int kFrameHeight = 720;
2631
2632 int64_t ntp_time = kFrameIntervalMs;
2633
2634 // Force an input frame rate to be available, or the adaptation call won't
2635 // know what framerate to adapt form.
2636 const int kInputFps = 30;
2637 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2638 stats.input_frame_rate = kInputFps;
2639 stats_proxy_->SetMockStats(stats);
2640
2641 video_source_.set_adaptation_enabled(true);
2642 video_stream_encoder_->SetSource(
2643 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002644 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002645 video_source_.IncomingCapturedFrame(
2646 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2647 sink_.WaitForEncodedFrame(ntp_time);
2648 ntp_time += kFrameIntervalMs;
2649
2650 // Trigger CPU overuse.
2651 video_stream_encoder_->TriggerCpuOveruse();
2652 video_source_.IncomingCapturedFrame(
2653 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2654 sink_.WaitForEncodedFrame(ntp_time);
2655 ntp_time += kFrameIntervalMs;
2656
2657 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2658 EXPECT_EQ(std::numeric_limits<int>::max(),
2659 video_source_.sink_wants().max_pixel_count);
2660 // Some framerate constraint should be set.
2661 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2662 EXPECT_LT(restricted_fps, kInputFps);
2663 video_source_.IncomingCapturedFrame(
2664 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2665 sink_.WaitForEncodedFrame(ntp_time);
2666 ntp_time += 100;
2667
Henrik Boström2671dac2020-05-19 16:29:09 +02002668 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002669 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2670 // Give the encoder queue time to process the change in degradation preference
2671 // by waiting for an encoded frame.
2672 video_source_.IncomingCapturedFrame(
2673 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2674 sink_.WaitForEncodedFrame(ntp_time);
2675 ntp_time += kFrameIntervalMs;
2676
2677 video_stream_encoder_->TriggerQualityLow();
2678 video_source_.IncomingCapturedFrame(
2679 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2680 sink_.WaitForEncodedFrame(ntp_time);
2681 ntp_time += kFrameIntervalMs;
2682
2683 // Some resolution constraint should be set.
2684 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2685 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2686 kFrameWidth * kFrameHeight);
2687 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2688
2689 int pixel_count = video_source_.sink_wants().max_pixel_count;
2690 // Triggering a CPU underuse should not change the sink wants since it has
2691 // not been overused for resolution since we changed degradation preference.
2692 video_stream_encoder_->TriggerCpuUnderuse();
2693 video_source_.IncomingCapturedFrame(
2694 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2695 sink_.WaitForEncodedFrame(ntp_time);
2696 ntp_time += kFrameIntervalMs;
2697 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2698 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2699
Evan Shrubsole64469032020-06-11 10:45:29 +02002700 // Change the degradation preference back. CPU underuse should not adapt since
2701 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002702 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002703 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2704 video_source_.IncomingCapturedFrame(
2705 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2706 sink_.WaitForEncodedFrame(ntp_time);
2707 ntp_time += 100;
2708 // Resolution adaptations is gone after changing degradation preference.
2709 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2710 EXPECT_EQ(std::numeric_limits<int>::max(),
2711 video_source_.sink_wants().max_pixel_count);
2712 // The fps adaptation from above is now back.
2713 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2714
2715 // Trigger CPU underuse.
2716 video_stream_encoder_->TriggerCpuUnderuse();
2717 video_source_.IncomingCapturedFrame(
2718 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2719 sink_.WaitForEncodedFrame(ntp_time);
2720 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002721 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2722
2723 // Trigger QP underuse, fps should return to normal.
2724 video_stream_encoder_->TriggerQualityHigh();
2725 video_source_.IncomingCapturedFrame(
2726 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2727 sink_.WaitForEncodedFrame(ntp_time);
2728 ntp_time += kFrameIntervalMs;
2729 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002730
2731 video_stream_encoder_->Stop();
2732}
2733
mflodmancc3d4422017-08-03 08:27:51 -07002734TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002736 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002737 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002738
sprangc5d62e22017-04-02 23:53:04 -07002739 const int kFrameWidth = 1280;
2740 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002741
Åsa Persson8c1bf952018-09-13 10:42:19 +02002742 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002743
kthelgason5e13d412016-12-01 03:59:51 -08002744 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002745 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002746 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002747 frame_timestamp += kFrameIntervalMs;
2748
perkj803d97f2016-11-01 11:45:46 -07002749 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002750 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002751 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002752 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002753 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002754 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002755
asapersson0944a802017-04-07 00:57:58 -07002756 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002757 // wanted resolution.
2758 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2759 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2760 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002761 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002762
2763 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002764 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002765 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002766 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002767 // Give the encoder queue time to process the change in degradation preference
2768 // by waiting for an encoded frame.
2769 new_video_source.IncomingCapturedFrame(
2770 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2771 sink_.WaitForEncodedFrame(frame_timestamp);
2772 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002773 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002774 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002775
sprangc5d62e22017-04-02 23:53:04 -07002776 // Force an input frame rate to be available, or the adaptation call won't
2777 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002778 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002779 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002780 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002781 stats_proxy_->SetMockStats(stats);
2782
mflodmancc3d4422017-08-03 08:27:51 -07002783 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002784 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002785 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002786 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002787 frame_timestamp += kFrameIntervalMs;
2788
2789 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002790 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002791 EXPECT_EQ(std::numeric_limits<int>::max(),
2792 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002793 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002794
asapersson02465b82017-04-10 01:12:52 -07002795 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002796 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2797 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002798 // Give the encoder queue time to process the change in degradation preference
2799 // by waiting for an encoded frame.
2800 new_video_source.IncomingCapturedFrame(
2801 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2802 sink_.WaitForEncodedFrame(frame_timestamp);
2803 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002804 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002805
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002807 new_video_source.IncomingCapturedFrame(
2808 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002809 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002810 frame_timestamp += kFrameIntervalMs;
2811
2812 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002813 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002814
2815 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002816 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002817 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002818 // Give the encoder queue time to process the change in degradation preference
2819 // by waiting for an encoded frame.
2820 new_video_source.IncomingCapturedFrame(
2821 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2822 sink_.WaitForEncodedFrame(frame_timestamp);
2823 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002824 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2825 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002826 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002827 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002828
2829 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002830 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002831 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002832 // Give the encoder queue time to process the change in degradation preference
2833 // by waiting for an encoded frame.
2834 new_video_source.IncomingCapturedFrame(
2835 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2836 sink_.WaitForEncodedFrame(frame_timestamp);
2837 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002838 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2839 EXPECT_EQ(std::numeric_limits<int>::max(),
2840 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002841 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002842
mflodmancc3d4422017-08-03 08:27:51 -07002843 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002844}
2845
mflodmancc3d4422017-08-03 08:27:51 -07002846TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002847 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002848 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002849
asaperssonfab67072017-04-04 05:51:49 -07002850 const int kWidth = 1280;
2851 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002852 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002853 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002854 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2855 EXPECT_FALSE(stats.bw_limited_resolution);
2856 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2857
2858 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002860 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002861 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002862
2863 stats = stats_proxy_->GetStats();
2864 EXPECT_TRUE(stats.bw_limited_resolution);
2865 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2866
2867 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002868 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002869 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002870 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002871
2872 stats = stats_proxy_->GetStats();
2873 EXPECT_FALSE(stats.bw_limited_resolution);
2874 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2875 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2876
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002878}
2879
mflodmancc3d4422017-08-03 08:27:51 -07002880TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002881 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002882 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002883
2884 const int kWidth = 1280;
2885 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002886 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002887 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002888 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2889 EXPECT_FALSE(stats.cpu_limited_resolution);
2890 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2891
2892 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002893 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002894 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002895 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002896
2897 stats = stats_proxy_->GetStats();
2898 EXPECT_TRUE(stats.cpu_limited_resolution);
2899 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2900
2901 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002902 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002903 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002904 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002905
2906 stats = stats_proxy_->GetStats();
2907 EXPECT_FALSE(stats.cpu_limited_resolution);
2908 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002909 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002910
mflodmancc3d4422017-08-03 08:27:51 -07002911 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002912}
2913
mflodmancc3d4422017-08-03 08:27:51 -07002914TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002915 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002916 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002917
asaperssonfab67072017-04-04 05:51:49 -07002918 const int kWidth = 1280;
2919 const int kHeight = 720;
2920 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002921 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002922 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002923 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002924 EXPECT_FALSE(stats.cpu_limited_resolution);
2925 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2926
asaperssonfab67072017-04-04 05:51:49 -07002927 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002929 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002930 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002931 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002932 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002933 EXPECT_TRUE(stats.cpu_limited_resolution);
2934 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2935
2936 // Set new source with adaptation still enabled.
2937 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002938 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002939 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002940
asaperssonfab67072017-04-04 05:51:49 -07002941 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002942 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002943 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002944 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002945 EXPECT_TRUE(stats.cpu_limited_resolution);
2946 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2947
2948 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002949 video_stream_encoder_->SetSource(&new_video_source,
2950 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002951
asaperssonfab67072017-04-04 05:51:49 -07002952 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002953 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002954 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002955 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002956 EXPECT_FALSE(stats.cpu_limited_resolution);
2957 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2958
2959 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002960 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002961 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002962
asaperssonfab67072017-04-04 05:51:49 -07002963 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002964 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002965 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002966 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002967 EXPECT_TRUE(stats.cpu_limited_resolution);
2968 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2969
asaperssonfab67072017-04-04 05:51:49 -07002970 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002971 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002972 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002973 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002974 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002975 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002976 EXPECT_FALSE(stats.cpu_limited_resolution);
2977 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002978 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002979
mflodmancc3d4422017-08-03 08:27:51 -07002980 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002981}
2982
mflodmancc3d4422017-08-03 08:27:51 -07002983TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002985 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002986
asaperssonfab67072017-04-04 05:51:49 -07002987 const int kWidth = 1280;
2988 const int kHeight = 720;
2989 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002990 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002991 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002992 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002993 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002994 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002995
2996 // Set new source with adaptation still enabled.
2997 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002998 video_stream_encoder_->SetSource(&new_video_source,
2999 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003000
asaperssonfab67072017-04-04 05:51:49 -07003001 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003002 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003003 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003004 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003005 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003006 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003007
asaperssonfab67072017-04-04 05:51:49 -07003008 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003009 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003010 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003011 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003012 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003013 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003014 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003015 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003016
asaperssonfab67072017-04-04 05:51:49 -07003017 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003018 video_stream_encoder_->SetSource(&new_video_source,
3019 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003020
asaperssonfab67072017-04-04 05:51:49 -07003021 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003022 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003023 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003024 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003025 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003026 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003027
asapersson02465b82017-04-10 01:12:52 -07003028 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003029 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003030 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003031
asaperssonfab67072017-04-04 05:51:49 -07003032 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003033 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003034 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003035 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003036 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003037 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3038 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003039
mflodmancc3d4422017-08-03 08:27:51 -07003040 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003041}
3042
mflodmancc3d4422017-08-03 08:27:51 -07003043TEST_F(VideoStreamEncoderTest,
3044 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003046 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003047
3048 const int kWidth = 1280;
3049 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003050 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003051 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003052 video_source_.IncomingCapturedFrame(
3053 CreateFrame(timestamp_ms, kWidth, kHeight));
3054 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003055 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3056 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3057 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3058
3059 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003060 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003061 timestamp_ms += kFrameIntervalMs;
3062 video_source_.IncomingCapturedFrame(
3063 CreateFrame(timestamp_ms, kWidth, kHeight));
3064 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003065 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3066 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3067 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3068
3069 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003070 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003071 timestamp_ms += kFrameIntervalMs;
3072 video_source_.IncomingCapturedFrame(
3073 CreateFrame(timestamp_ms, kWidth, kHeight));
3074 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3077 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3078
Niels Möller4db138e2018-04-19 09:04:13 +02003079 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003080 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003081
3082 VideoEncoderConfig video_encoder_config;
3083 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3084 // Make format different, to force recreation of encoder.
3085 video_encoder_config.video_format.parameters["foo"] = "foo";
3086 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003087 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003088 timestamp_ms += kFrameIntervalMs;
3089 video_source_.IncomingCapturedFrame(
3090 CreateFrame(timestamp_ms, kWidth, kHeight));
3091 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003092 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3093 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3094 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3095
mflodmancc3d4422017-08-03 08:27:51 -07003096 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003097}
3098
mflodmancc3d4422017-08-03 08:27:51 -07003099TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003100 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003101 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003102 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003103
3104 const int kWidth = 1280;
3105 const int kHeight = 720;
3106 int sequence = 1;
3107
3108 // Enable BALANCED preference, no initial limitation.
3109 test::FrameForwarder source;
3110 video_stream_encoder_->SetSource(&source,
3111 webrtc::DegradationPreference::BALANCED);
3112 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3113 WaitForEncodedFrame(sequence++);
3114 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3115 EXPECT_FALSE(stats.cpu_limited_resolution);
3116 EXPECT_FALSE(stats.cpu_limited_framerate);
3117 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3118
3119 // Trigger CPU overuse, should now adapt down.
3120 video_stream_encoder_->TriggerCpuOveruse();
3121 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3122 WaitForEncodedFrame(sequence++);
3123 stats = stats_proxy_->GetStats();
3124 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3125
3126 // Set new degradation preference should clear restrictions since we changed
3127 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003128 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003129 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3130 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3131 WaitForEncodedFrame(sequence++);
3132 stats = stats_proxy_->GetStats();
3133 EXPECT_FALSE(stats.cpu_limited_resolution);
3134 EXPECT_FALSE(stats.cpu_limited_framerate);
3135 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3136
3137 // Force an input frame rate to be available, or the adaptation call won't
3138 // know what framerate to adapt from.
3139 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3140 mock_stats.input_frame_rate = 30;
3141 stats_proxy_->SetMockStats(mock_stats);
3142 video_stream_encoder_->TriggerCpuOveruse();
3143 stats_proxy_->ResetMockStats();
3144 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3145 WaitForEncodedFrame(sequence++);
3146
3147 // We have now adapted once.
3148 stats = stats_proxy_->GetStats();
3149 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3150
3151 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003152 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3153 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003154 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3155 WaitForEncodedFrame(sequence++);
3156 stats = stats_proxy_->GetStats();
3157 EXPECT_FALSE(stats.cpu_limited_resolution);
3158 EXPECT_FALSE(stats.cpu_limited_framerate);
3159 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3160
3161 video_stream_encoder_->Stop();
3162}
3163
3164TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003165 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003166 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003167 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003168
asapersson0944a802017-04-07 00:57:58 -07003169 const int kWidth = 1280;
3170 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003171 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003172
asaperssonfab67072017-04-04 05:51:49 -07003173 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003174 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003175 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003176 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003177 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003178 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3179
asapersson02465b82017-04-10 01:12:52 -07003180 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003181 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003182 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003183 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003184 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003185 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003186 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003187 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3188
3189 // Set new source with adaptation still enabled.
3190 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003191 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003192 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003193
3194 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003195 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003196 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003197 stats = stats_proxy_->GetStats();
3198 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003199 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003200 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3201
sprangc5d62e22017-04-02 23:53:04 -07003202 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003203 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003204 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003205 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003206 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003207 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003208 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003209 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003210 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003211 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003212 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3213
sprangc5d62e22017-04-02 23:53:04 -07003214 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003215 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003216 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3217 mock_stats.input_frame_rate = 30;
3218 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003219 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003220 stats_proxy_->ResetMockStats();
3221
3222 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003223 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003224 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003225
3226 // Framerate now adapted.
3227 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003228 EXPECT_FALSE(stats.cpu_limited_resolution);
3229 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003230 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3231
3232 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003233 video_stream_encoder_->SetSource(&new_video_source,
3234 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003235 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003236 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003237 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003238
3239 stats = stats_proxy_->GetStats();
3240 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003241 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003242 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3243
3244 // Try to trigger overuse. Should not succeed.
3245 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003246 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003247 stats_proxy_->ResetMockStats();
3248
3249 stats = stats_proxy_->GetStats();
3250 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003251 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003252 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3253
3254 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003256 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003257 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003258 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003259 stats = stats_proxy_->GetStats();
3260 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003261 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003262 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003263
3264 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003265 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003266 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003267 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003268 stats = stats_proxy_->GetStats();
3269 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003270 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003271 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3272
3273 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003274 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003275 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003276 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003277 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003278 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003279 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003280 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003281 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003282 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003283 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3284
3285 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003286 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003287 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003288 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003289 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003290 stats = stats_proxy_->GetStats();
3291 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003292 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003293 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003294 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003295
mflodmancc3d4422017-08-03 08:27:51 -07003296 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003297}
3298
mflodmancc3d4422017-08-03 08:27:51 -07003299TEST_F(VideoStreamEncoderTest,
3300 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003301 const int kWidth = 1280;
3302 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003304 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003305
asaperssonfab67072017-04-04 05:51:49 -07003306 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003307 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003308
asaperssonfab67072017-04-04 05:51:49 -07003309 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003310 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003311
asaperssonfab67072017-04-04 05:51:49 -07003312 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003313 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003314
asaperssonfab67072017-04-04 05:51:49 -07003315 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003316 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003317
kthelgason876222f2016-11-29 01:44:11 -08003318 // Expect a scale down.
3319 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003320 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003321
asapersson02465b82017-04-10 01:12:52 -07003322 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003323 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003324 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003325 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003326
asaperssonfab67072017-04-04 05:51:49 -07003327 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003328 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003329 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003330 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003331
asaperssonfab67072017-04-04 05:51:49 -07003332 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003333 EXPECT_EQ(std::numeric_limits<int>::max(),
3334 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003335
asaperssonfab67072017-04-04 05:51:49 -07003336 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003337 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003338 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003339 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003340
asapersson02465b82017-04-10 01:12:52 -07003341 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003342 EXPECT_EQ(std::numeric_limits<int>::max(),
3343 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003344
mflodmancc3d4422017-08-03 08:27:51 -07003345 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003346}
3347
mflodmancc3d4422017-08-03 08:27:51 -07003348TEST_F(VideoStreamEncoderTest,
3349 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003350 const int kWidth = 1280;
3351 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003353 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003354
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003355 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003356 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003357 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003358 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003359
3360 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003361 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003362 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003363 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3364 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3365
3366 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003367 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003368 EXPECT_THAT(source.sink_wants(),
3369 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003370 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3371 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3372 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3373
3374 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003375 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003376 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3377 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3378 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3379
mflodmancc3d4422017-08-03 08:27:51 -07003380 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003381}
3382
mflodmancc3d4422017-08-03 08:27:51 -07003383TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003384 const int kWidth = 1280;
3385 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003387 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003388
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003389 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003390 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003391 video_stream_encoder_->SetSource(&source,
3392 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003393 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3394 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003395 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003396
3397 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003398 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003399 EXPECT_THAT(source.sink_wants(),
3400 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003401 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3402 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3403 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3404
3405 // Trigger adapt down for same input resolution, expect no change.
3406 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3407 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003408 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003409 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3410 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3411 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3412
3413 // Trigger adapt down for larger input resolution, expect no change.
3414 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3415 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003416 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003417 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3418 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3419 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3420
mflodmancc3d4422017-08-03 08:27:51 -07003421 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003422}
3423
mflodmancc3d4422017-08-03 08:27:51 -07003424TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003425 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3426 const int kWidth = 640;
3427 const int kHeight = 360;
3428 const int64_t kFrameIntervalMs = 150;
3429 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003430 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003431
3432 // Enable BALANCED preference, no initial limitation.
3433 AdaptingFrameForwarder source(&time_controller_);
3434 source.set_adaptation_enabled(true);
3435 video_stream_encoder_->SetSource(&source,
3436 webrtc::DegradationPreference::BALANCED);
3437
3438 int64_t timestamp_ms = kFrameIntervalMs;
3439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3440 sink_.WaitForEncodedFrame(kWidth, kHeight);
3441 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3442 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3445
3446 // Trigger adapt down, expect reduced fps (640x360@15fps).
3447 video_stream_encoder_->TriggerQualityLow();
3448 timestamp_ms += kFrameIntervalMs;
3449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3450 sink_.WaitForEncodedFrame(timestamp_ms);
3451 EXPECT_THAT(source.sink_wants(),
3452 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3455 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3456
3457 // Source requests 270p, expect reduced resolution (480x270@15fps).
3458 source.OnOutputFormatRequest(480, 270);
3459 timestamp_ms += kFrameIntervalMs;
3460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3461 WaitForEncodedFrame(480, 270);
3462 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3463
3464 // Trigger adapt down, expect reduced fps (480x270@10fps).
3465 video_stream_encoder_->TriggerQualityLow();
3466 timestamp_ms += kFrameIntervalMs;
3467 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3468 sink_.WaitForEncodedFrame(timestamp_ms);
3469 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3470 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3471 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3472 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3473
3474 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3475 source.OnOutputFormatRequest(320, 180);
3476 timestamp_ms += kFrameIntervalMs;
3477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3478 WaitForEncodedFrame(320, 180);
3479 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3480
3481 // Trigger adapt down, expect reduced fps (320x180@7fps).
3482 video_stream_encoder_->TriggerQualityLow();
3483 timestamp_ms += kFrameIntervalMs;
3484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3485 sink_.WaitForEncodedFrame(timestamp_ms);
3486 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3489 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3490
3491 // Source requests VGA, expect increased resolution (640x360@7fps).
3492 source.OnOutputFormatRequest(640, 360);
3493 timestamp_ms += kFrameIntervalMs;
3494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3495 WaitForEncodedFrame(timestamp_ms);
3496 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3497
3498 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3499 video_stream_encoder_->TriggerQualityHigh();
3500 timestamp_ms += kFrameIntervalMs;
3501 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3502 WaitForEncodedFrame(timestamp_ms);
3503 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3504 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3506 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3507
3508 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3509 video_stream_encoder_->TriggerQualityHigh();
3510 timestamp_ms += kFrameIntervalMs;
3511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3512 WaitForEncodedFrame(timestamp_ms);
3513 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3516 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3517
3518 // Trigger adapt up, expect increased fps (640x360@maxfps).
3519 video_stream_encoder_->TriggerQualityHigh();
3520 timestamp_ms += kFrameIntervalMs;
3521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3522 WaitForEncodedFrame(timestamp_ms);
3523 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3526 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3527
3528 video_stream_encoder_->Stop();
3529}
3530
3531TEST_F(VideoStreamEncoderTest,
3532 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3533 const int kWidth = 1280;
3534 const int kHeight = 720;
3535 const int64_t kFrameIntervalMs = 150;
3536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003537 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003538
3539 // Enable BALANCED preference, no initial limitation.
3540 AdaptingFrameForwarder source(&time_controller_);
3541 source.set_adaptation_enabled(true);
3542 video_stream_encoder_->SetSource(&source,
3543 webrtc::DegradationPreference::BALANCED);
3544
3545 int64_t timestamp_ms = kFrameIntervalMs;
3546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3547 sink_.WaitForEncodedFrame(kWidth, kHeight);
3548 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3551 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3552
3553 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3554 video_stream_encoder_->TriggerQualityLow();
3555 timestamp_ms += kFrameIntervalMs;
3556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3557 sink_.WaitForEncodedFrame(timestamp_ms);
3558 EXPECT_THAT(source.sink_wants(),
3559 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3561 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3562 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3563
3564 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3565 video_stream_encoder_->TriggerQualityLow();
3566 timestamp_ms += kFrameIntervalMs;
3567 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3568 sink_.WaitForEncodedFrame(timestamp_ms);
3569 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3571 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3572 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3573
3574 // Trigger adapt down, expect reduced fps (640x360@15fps).
3575 video_stream_encoder_->TriggerQualityLow();
3576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 WaitForEncodedFrame(timestamp_ms);
3579 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3582 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3583
3584 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3585 source.OnOutputFormatRequest(320, 180);
3586 timestamp_ms += kFrameIntervalMs;
3587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3588 WaitForEncodedFrame(320, 180);
3589 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3590 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3591
3592 // Trigger adapt down, expect reduced fps (320x180@7fps).
3593 video_stream_encoder_->TriggerCpuOveruse();
3594 timestamp_ms += kFrameIntervalMs;
3595 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3596 WaitForEncodedFrame(timestamp_ms);
3597 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3599 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3600 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3601 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3602 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3603 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3604
3605 // Source requests HD, expect increased resolution (640x360@7fps).
3606 source.OnOutputFormatRequest(1280, 720);
3607 timestamp_ms += kFrameIntervalMs;
3608 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3609 WaitForEncodedFrame(timestamp_ms);
3610 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3611 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3612
3613 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3614 video_stream_encoder_->TriggerCpuUnderuse();
3615 timestamp_ms += kFrameIntervalMs;
3616 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3617 WaitForEncodedFrame(timestamp_ms);
3618 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3621 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3622 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3623 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3624 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3625
3626 // Trigger adapt up, expect increased fps (640x360@maxfps).
3627 video_stream_encoder_->TriggerQualityHigh();
3628 video_stream_encoder_->TriggerCpuUnderuse();
3629 timestamp_ms += kFrameIntervalMs;
3630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3631 WaitForEncodedFrame(timestamp_ms);
3632 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3633 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3634 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3635 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3636 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3637 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3638 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3639
3640 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3641 video_stream_encoder_->TriggerQualityHigh();
3642 video_stream_encoder_->TriggerCpuUnderuse();
3643 timestamp_ms += kFrameIntervalMs;
3644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3645 WaitForEncodedFrame(timestamp_ms);
3646 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3649 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3650 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3651 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3652 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3653
3654 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3655 video_stream_encoder_->TriggerQualityHigh();
3656 video_stream_encoder_->TriggerCpuUnderuse();
3657 timestamp_ms += kFrameIntervalMs;
3658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3659 WaitForEncodedFrame(timestamp_ms);
3660 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3661 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3663 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3664 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3665 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3666 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3667
3668 video_stream_encoder_->Stop();
3669}
3670
3671TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003672 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003673 const int kWidth = 1280;
3674 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003676 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003677
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003678 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003679 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003680 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003681 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003682
3683 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003684 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003685 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003686 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3687 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3688
3689 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003690 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003691 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003692 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3693 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3694
mflodmancc3d4422017-08-03 08:27:51 -07003695 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003696}
3697
mflodmancc3d4422017-08-03 08:27:51 -07003698TEST_F(VideoStreamEncoderTest,
3699 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003700 const int kWidth = 1280;
3701 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003703 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003704
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003705 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003706 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003707 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003708 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003709
3710 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003711 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003712 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003713 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003714 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3715
3716 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003717 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003719 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3721
mflodmancc3d4422017-08-03 08:27:51 -07003722 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003723}
3724
mflodmancc3d4422017-08-03 08:27:51 -07003725TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003726 const int kWidth = 1280;
3727 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003729 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003730
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003731 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003732 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003733 video_stream_encoder_->SetSource(&source,
3734 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003735
3736 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3737 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003738 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003739 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3740 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3741 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3742
3743 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003744 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003745 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3747 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3748 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3749
mflodmancc3d4422017-08-03 08:27:51 -07003750 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003751}
3752
mflodmancc3d4422017-08-03 08:27:51 -07003753TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003754 const int kWidth = 1280;
3755 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003757 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003758
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003759 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003760 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003761 video_stream_encoder_->SetSource(&source,
3762 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003763
3764 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3765 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003766 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003767 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3768 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3769 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3770
3771 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003772 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003773 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003774 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3775 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3776 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3777
mflodmancc3d4422017-08-03 08:27:51 -07003778 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003779}
3780
mflodmancc3d4422017-08-03 08:27:51 -07003781TEST_F(VideoStreamEncoderTest,
3782 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003783 const int kWidth = 1280;
3784 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003786 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003787
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003788 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003789 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003790 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003791 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003792 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003793
3794 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003795 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003796 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3798 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3799
3800 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003801 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003802 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003803 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003804 EXPECT_THAT(source.sink_wants(),
3805 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003806 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3807 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3808
3809 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003810 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003811 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003812 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3813 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3814 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3815
mflodmancc3d4422017-08-03 08:27:51 -07003816 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003817}
3818
mflodmancc3d4422017-08-03 08:27:51 -07003819TEST_F(VideoStreamEncoderTest,
3820 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003821 const int kWidth = 1280;
3822 const int kHeight = 720;
3823 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003824 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003825 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003826
3827 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3828 stats.input_frame_rate = kInputFps;
3829 stats_proxy_->SetMockStats(stats);
3830
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003831 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003832 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3833 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003834 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003835
3836 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003837 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003838 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3839 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003840 EXPECT_THAT(video_source_.sink_wants(),
3841 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003843 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003844 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003845 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003846 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003847 // Give the encoder queue time to process the change in degradation preference
3848 // by waiting for an encoded frame.
3849 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3850 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003851 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003852
3853 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003854 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003855 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3856 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003857 EXPECT_THAT(new_video_source.sink_wants(),
3858 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003859
3860 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003861 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003862 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003863
mflodmancc3d4422017-08-03 08:27:51 -07003864 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003865}
3866
mflodmancc3d4422017-08-03 08:27:51 -07003867TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003868 const int kWidth = 1280;
3869 const int kHeight = 720;
3870 const size_t kNumFrames = 10;
3871
Henrik Boström381d1092020-05-12 18:49:07 +02003872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003873 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003874
asaperssond0de2952017-04-21 01:47:31 -07003875 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003876 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003877 video_source_.set_adaptation_enabled(true);
3878
3879 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3880 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3881
3882 int downscales = 0;
3883 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003884 video_source_.IncomingCapturedFrame(
3885 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3886 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003887
asaperssonfab67072017-04-04 05:51:49 -07003888 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003889 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003890 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003891 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003892
3893 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3894 ++downscales;
3895
3896 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3897 EXPECT_EQ(downscales,
3898 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3899 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003900 }
mflodmancc3d4422017-08-03 08:27:51 -07003901 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003902}
3903
mflodmancc3d4422017-08-03 08:27:51 -07003904TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003905 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3906 const int kWidth = 1280;
3907 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003908 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003909 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003910
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003911 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003912 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003913 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003914 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003915 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003916
Åsa Persson8c1bf952018-09-13 10:42:19 +02003917 int64_t timestamp_ms = kFrameIntervalMs;
3918 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003919 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003920 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3922 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3923
3924 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003926 timestamp_ms += kFrameIntervalMs;
3927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3928 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003929 EXPECT_THAT(source.sink_wants(),
3930 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003931 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3932 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3933
3934 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003935 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003936 timestamp_ms += kFrameIntervalMs;
3937 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003938 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003939 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003940 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3941 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3942
3943 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003944 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003945 timestamp_ms += kFrameIntervalMs;
3946 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3947 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003948 EXPECT_THAT(source.sink_wants(),
3949 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003950 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3951 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3952
3953 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003954 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003955 timestamp_ms += kFrameIntervalMs;
3956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003957 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003958 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003959 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3960 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3961
mflodmancc3d4422017-08-03 08:27:51 -07003962 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003963}
3964
mflodmancc3d4422017-08-03 08:27:51 -07003965TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003966 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3967 const int kWidth = 1280;
3968 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003970 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003971
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003972 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003973 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003974 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003975 video_stream_encoder_->SetSource(&source,
3976 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003977
Åsa Persson8c1bf952018-09-13 10:42:19 +02003978 int64_t timestamp_ms = kFrameIntervalMs;
3979 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003980 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003981 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003982 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3983 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3984
3985 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003986 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003987 timestamp_ms += kFrameIntervalMs;
3988 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3989 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003990 EXPECT_THAT(source.sink_wants(),
3991 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003992 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3993 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3994
3995 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003996 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003997 timestamp_ms += kFrameIntervalMs;
3998 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003999 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004000 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4002 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4003
4004 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004005 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004006 timestamp_ms += kFrameIntervalMs;
4007 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4008 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004009 EXPECT_THAT(source.sink_wants(),
4010 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004011 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4012 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4013
4014 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004015 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004016 timestamp_ms += kFrameIntervalMs;
4017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004018 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004019 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004020 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4021 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4022
mflodmancc3d4422017-08-03 08:27:51 -07004023 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004024}
4025
Sergey Silkin41c650b2019-10-14 13:12:19 +02004026TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4027 fake_encoder_.SetResolutionBitrateLimits(
4028 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4029
Henrik Boström381d1092020-05-12 18:49:07 +02004030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004031 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4032 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4033 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4034 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004035
4036 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004037 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004038 source.set_adaptation_enabled(true);
4039 video_stream_encoder_->SetSource(
4040 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4041
4042 // Insert 720p frame.
4043 int64_t timestamp_ms = kFrameIntervalMs;
4044 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4045 WaitForEncodedFrame(1280, 720);
4046
4047 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004049 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4050 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4051 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4052 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004053 video_stream_encoder_->TriggerQualityLow();
4054
4055 // Insert 720p frame. It should be downscaled and encoded.
4056 timestamp_ms += kFrameIntervalMs;
4057 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4058 WaitForEncodedFrame(960, 540);
4059
4060 // Trigger adapt up. Higher resolution should not be requested duo to lack
4061 // of bitrate.
4062 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004063 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004064
4065 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004066 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004067 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4068 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4069 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4070 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004071
4072 // Trigger adapt up. Higher resolution should be requested.
4073 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004074 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004075
4076 video_stream_encoder_->Stop();
4077}
4078
4079TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4080 fake_encoder_.SetResolutionBitrateLimits(
4081 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4082
4083 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004084 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004085 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4086 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4087 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4088 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004089
4090 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004091 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004092 source.set_adaptation_enabled(true);
4093 video_stream_encoder_->SetSource(
4094 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4095
4096 // Insert 720p frame. It should be dropped and lower resolution should be
4097 // requested.
4098 int64_t timestamp_ms = kFrameIntervalMs;
4099 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4100 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004101 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004102
4103 // Insert 720p frame. It should be downscaled and encoded.
4104 timestamp_ms += kFrameIntervalMs;
4105 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4106 WaitForEncodedFrame(960, 540);
4107
4108 video_stream_encoder_->Stop();
4109}
4110
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004111class BalancedDegradationTest : public VideoStreamEncoderTest {
4112 protected:
4113 void SetupTest() {
4114 // Reset encoder for field trials to take effect.
4115 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004116 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004117
4118 // Enable BALANCED preference.
4119 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004120 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4121 }
4122
Asa Persson606d3cb2021-10-04 10:07:11 +02004123 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004125 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004126 }
4127
Åsa Persson45b176f2019-09-30 11:19:05 +02004128 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004129 timestamp_ms_ += kFrameIntervalMs;
4130 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004131 }
4132
4133 void InsertFrameAndWaitForEncoded() {
4134 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004135 sink_.WaitForEncodedFrame(timestamp_ms_);
4136 }
4137
4138 const int kWidth = 640; // pixels:640x360=230400
4139 const int kHeight = 360;
4140 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4141 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004142 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004143};
4144
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004145TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004146 test::ScopedKeyValueConfig field_trials(
4147 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004148 "WebRTC-Video-BalancedDegradationSettings/"
4149 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4150 SetupTest();
4151
4152 // Force input frame rate.
4153 const int kInputFps = 24;
4154 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4155 stats.input_frame_rate = kInputFps;
4156 stats_proxy_->SetMockStats(stats);
4157
Åsa Persson45b176f2019-09-30 11:19:05 +02004158 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004159 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004160
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004161 // Trigger adapt down, expect scaled down framerate and resolution,
4162 // since Fps diff (input-requested:0) < threshold.
4163 video_stream_encoder_->TriggerQualityLow();
4164 EXPECT_THAT(source_.sink_wants(),
4165 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004166
4167 video_stream_encoder_->Stop();
4168}
4169
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004170TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004171 test::ScopedKeyValueConfig field_trials(
4172 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004173 "WebRTC-Video-BalancedDegradationSettings/"
4174 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4175 SetupTest();
4176
4177 // Force input frame rate.
4178 const int kInputFps = 25;
4179 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4180 stats.input_frame_rate = kInputFps;
4181 stats_proxy_->SetMockStats(stats);
4182
Åsa Persson45b176f2019-09-30 11:19:05 +02004183 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004184 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004185
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004186 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4187 // Fps diff (input-requested:1) == threshold.
4188 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004189 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004190
4191 video_stream_encoder_->Stop();
4192}
4193
4194TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004195 test::ScopedKeyValueConfig field_trials(
4196 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004197 "WebRTC-Video-BalancedDegradationSettings/"
4198 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4199 SetupTest();
4200
4201 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4202
Åsa Persson45b176f2019-09-30 11:19:05 +02004203 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004204 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004205
4206 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4207 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004208 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004209
4210 video_stream_encoder_->Stop();
4211}
4212
Åsa Perssonccfb3402019-09-25 15:13:04 +02004213TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004214 test::ScopedKeyValueConfig field_trials(
4215 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004216 "WebRTC-Video-BalancedDegradationSettings/"
4217 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004218 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004219
Asa Persson606d3cb2021-10-04 10:07:11 +02004220 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4221 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4222 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004223
Åsa Persson45b176f2019-09-30 11:19:05 +02004224 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004225 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004226 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4227
4228 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4229 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004230 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004231 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004232 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4233
4234 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4235 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004236 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004237 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004238 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4239
Åsa Persson30ab0152019-08-27 12:22:33 +02004240 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4241 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004242 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004243 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004244 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004245 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4246
4247 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004248 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004249 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004250 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004251
Åsa Persson30ab0152019-08-27 12:22:33 +02004252 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004253 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004254 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004255 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004256 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004257 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4258
4259 video_stream_encoder_->Stop();
4260}
4261
Åsa Perssonccfb3402019-09-25 15:13:04 +02004262TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004263 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004264 test::ScopedKeyValueConfig field_trials(
4265 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004266 "WebRTC-Video-BalancedDegradationSettings/"
4267 "pixels:57600|129600|230400,fps:7|24|24/");
4268 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004269 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004270
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004271 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004272
4273 // Insert frame, expect scaled down:
4274 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4275 InsertFrame();
4276 EXPECT_FALSE(WaitForFrame(1000));
4277 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4278 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4279
4280 // Insert frame, expect scaled down:
4281 // resolution (320x180@24fps).
4282 InsertFrame();
4283 EXPECT_FALSE(WaitForFrame(1000));
4284 EXPECT_LT(source_.sink_wants().max_pixel_count,
4285 source_.last_wants().max_pixel_count);
4286 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4287
4288 // Frame should not be dropped (min pixels per frame reached).
4289 InsertFrameAndWaitForEncoded();
4290
4291 video_stream_encoder_->Stop();
4292}
4293
4294TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004295 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004296 test::ScopedKeyValueConfig field_trials(
4297 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004298 "WebRTC-Video-BalancedDegradationSettings/"
4299 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004300 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004301
Asa Persson606d3cb2021-10-04 10:07:11 +02004302 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4303 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4304 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004305
Åsa Persson45b176f2019-09-30 11:19:05 +02004306 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004307 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004308 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4309
4310 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4311 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004312 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004313 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004314 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4315
4316 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4317 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004318 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004319 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004320 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4321
4322 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4323 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004324 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004325 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004326 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4327
Åsa Persson30ab0152019-08-27 12:22:33 +02004328 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4329 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004330 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004331 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004332 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4333
4334 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4335 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004336 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004337 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4338
4339 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004340 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004341 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004342 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004343 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 video_stream_encoder_->Stop();
4347}
4348
Åsa Perssonccfb3402019-09-25 15:13:04 +02004349TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004350 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004351 test::ScopedKeyValueConfig field_trials(
4352 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004353 "WebRTC-Video-BalancedDegradationSettings/"
4354 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004355 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004356
Asa Persson606d3cb2021-10-04 10:07:11 +02004357 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4358 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4359 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4360 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4361 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004362
Åsa Persson45b176f2019-09-30 11:19:05 +02004363 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004364 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004365 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4366
4367 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4368 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004369 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004370 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004371 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4372
4373 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4374 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004375 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004376 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004377 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4378
4379 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4380 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004381 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004382 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004383 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4384
4385 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4386 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004387 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004388 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4389
4390 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004391 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004392 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004393 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004394 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004395 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4396
4397 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004398 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004399 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004400 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004401 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4402
4403 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004404 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004405 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004406 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004407 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004408 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4409
Åsa Persson1b247f12019-08-14 17:26:39 +02004410 video_stream_encoder_->Stop();
4411}
4412
mflodmancc3d4422017-08-03 08:27:51 -07004413TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004414 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4415 const int kWidth = 1280;
4416 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004418 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004419
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004420 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004421 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004422 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004423 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004424 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004425
Åsa Persson8c1bf952018-09-13 10:42:19 +02004426 int64_t timestamp_ms = kFrameIntervalMs;
4427 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004428 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004429 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004430 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4431 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4432 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4433 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4434
4435 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004436 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004437 timestamp_ms += kFrameIntervalMs;
4438 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4439 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004440 EXPECT_THAT(source.sink_wants(),
4441 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004442 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4444 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4445 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4446
4447 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004448 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004449 timestamp_ms += kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4451 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004452 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004453 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4454 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4455 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4457
Jonathan Yubc771b72017-12-08 17:04:29 -08004458 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004459 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004460 timestamp_ms += kFrameIntervalMs;
4461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4462 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004463 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004464 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004466 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004467 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4468
Jonathan Yubc771b72017-12-08 17:04:29 -08004469 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004470 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004471 timestamp_ms += kFrameIntervalMs;
4472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4473 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004474 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004475 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004476 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4477 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4478 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4479 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4480
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 // Trigger quality adapt down, expect no change (min resolution reached).
4482 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004483 timestamp_ms += kFrameIntervalMs;
4484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004486 EXPECT_THAT(source.sink_wants(), FpsMax());
4487 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4490 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4491 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4492
Evan Shrubsole64469032020-06-11 10:45:29 +02004493 // Trigger quality adapt up, expect upscaled resolution (480x270).
4494 video_stream_encoder_->TriggerQualityHigh();
4495 timestamp_ms += kFrameIntervalMs;
4496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4497 WaitForEncodedFrame(timestamp_ms);
4498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4499 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4501 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4503
4504 // Trigger quality and cpu adapt up since both are most limited, expect
4505 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004506 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004507 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004508 timestamp_ms += kFrameIntervalMs;
4509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4510 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004511 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004512 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4514 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004515 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004516
Evan Shrubsole64469032020-06-11 10:45:29 +02004517 // Trigger quality and cpu adapt up since both are most limited, expect
4518 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004519 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004520 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004521 timestamp_ms += kFrameIntervalMs;
4522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4523 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004524 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004525 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004526 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004528 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4529 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004530
Evan Shrubsole64469032020-06-11 10:45:29 +02004531 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4532 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004533 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004534 timestamp_ms += kFrameIntervalMs;
4535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4536 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004537 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004538 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004540 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004541 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004542
4543 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004544 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004545 timestamp_ms += kFrameIntervalMs;
4546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004547 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004548 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004549 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004550 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004552 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004553 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004554
mflodmancc3d4422017-08-03 08:27:51 -07004555 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004556}
4557
mflodmancc3d4422017-08-03 08:27:51 -07004558TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004559 const int kWidth = 640;
4560 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004561
Henrik Boström381d1092020-05-12 18:49:07 +02004562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004563 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004564
perkj803d97f2016-11-01 11:45:46 -07004565 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004566 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004567 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004568 }
4569
mflodmancc3d4422017-08-03 08:27:51 -07004570 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004571 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004572 video_source_.IncomingCapturedFrame(CreateFrame(
4573 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004574 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004575 }
4576
mflodmancc3d4422017-08-03 08:27:51 -07004577 video_stream_encoder_->Stop();
4578 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004579 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004580
Ying Wangef3998f2019-12-09 13:06:53 +01004581 EXPECT_METRIC_EQ(
4582 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4583 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004584 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4585}
4586
mflodmancc3d4422017-08-03 08:27:51 -07004587TEST_F(VideoStreamEncoderTest,
4588 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004589 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004590 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004591 const int kWidth = 640;
4592 const int kHeight = 360;
4593
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004594 video_stream_encoder_->SetSource(&video_source_,
4595 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004596
4597 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4598 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004599 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004600 }
4601
mflodmancc3d4422017-08-03 08:27:51 -07004602 video_stream_encoder_->Stop();
4603 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004604 stats_proxy_.reset();
4605
4606 EXPECT_EQ(0,
4607 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4608}
4609
Per Kjellanderdcef6412020-10-07 15:09:05 +02004610TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4611 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004612 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004613 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004614
4615 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004616 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004617 SimulcastRateAllocator(fake_encoder_.config())
4618 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004619 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004620
Henrik Boström381d1092020-05-12 18:49:07 +02004621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004622 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004623
sprang57c2fff2017-01-16 06:24:02 -08004624 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004625 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4626 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004627 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4628 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4629
Erik Språngd7329ca2019-02-21 21:19:53 +01004630 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004631 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004632 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004633
Per Kjellanderdcef6412020-10-07 15:09:05 +02004634 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004635 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004636 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4637 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004638 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004639 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004640
Per Kjellanderdcef6412020-10-07 15:09:05 +02004641 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004642 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004643 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004644 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004645 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4646 WaitForEncodedFrame(CurrentTimeMs());
4647 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004648 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004649 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004650
mflodmancc3d4422017-08-03 08:27:51 -07004651 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004652}
4653
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004654TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004655 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004656 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004657 kVideoLayersAllocation);
4658
4659 const int kDefaultFps = 30;
4660
4661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004662 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004663
4664 video_source_.IncomingCapturedFrame(
4665 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4666 WaitForEncodedFrame(CurrentTimeMs());
4667 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4668 VideoLayersAllocation last_layer_allocation =
4669 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004670 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004671 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4672
4673 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004674 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004675 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004676 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004677 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4678
Erik Språng9d69cbe2020-10-22 17:44:42 +02004679 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004680 int number_of_layers_allocation = 1;
4681 const int64_t start_time_ms = CurrentTimeMs();
4682 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4683 video_source_.IncomingCapturedFrame(
4684 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4685 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004686 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4687 number_of_layers_allocation = sink_.number_of_layers_allocations();
4688 VideoLayersAllocation new_allocation =
4689 sink_.GetLastVideoLayersAllocation();
4690 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4691 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4692 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4693 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4694 .target_bitrate_per_temporal_layer,
4695 last_layer_allocation.active_spatial_layers[0]
4696 .target_bitrate_per_temporal_layer);
4697 last_layer_allocation = new_allocation;
4698 }
4699 }
4700 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4701 video_stream_encoder_->Stop();
4702}
4703
4704TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004705 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004706 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4707 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4708 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004709 VideoEncoderConfig video_encoder_config;
4710 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4711 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004712 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004713 video_encoder_config.content_type =
4714 VideoEncoderConfig::ContentType::kRealtimeVideo;
4715 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004716 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004717 VideoEncoder::GetDefaultVp8Settings());
4718 for (auto& layer : video_encoder_config.simulcast_layers) {
4719 layer.num_temporal_layers = 2;
4720 }
4721 // Simulcast layers are used for enabling/disabling streams.
4722 video_encoder_config.simulcast_layers[0].active = true;
4723 video_encoder_config.simulcast_layers[1].active = false;
4724 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004725 ConfigureEncoder(std::move(video_encoder_config),
4726 VideoStreamEncoder::BitrateAllocationCallbackType::
4727 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004728
4729 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004730 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004731
4732 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4733 WaitForEncodedFrame(CurrentTimeMs());
4734 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4735 VideoLayersAllocation last_layer_allocation =
4736 sink_.GetLastVideoLayersAllocation();
4737
4738 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4739 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4740 .target_bitrate_per_temporal_layer,
4741 SizeIs(2));
4742 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4743 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4744 video_stream_encoder_->Stop();
4745}
4746
4747TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004748 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004749 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4750 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4751 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004752 VideoEncoderConfig video_encoder_config;
4753 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4754 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004755 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004756 video_encoder_config.content_type =
4757 VideoEncoderConfig::ContentType::kRealtimeVideo;
4758 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004759 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004760 VideoEncoder::GetDefaultVp8Settings());
4761 for (auto& layer : video_encoder_config.simulcast_layers) {
4762 layer.num_temporal_layers = 2;
4763 }
4764 // Simulcast layers are used for enabling/disabling streams.
4765 video_encoder_config.simulcast_layers[0].active = true;
4766 video_encoder_config.simulcast_layers[1].active = false;
4767 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004768 ConfigureEncoder(std::move(video_encoder_config),
4769 VideoStreamEncoder::BitrateAllocationCallbackType::
4770 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004771
4772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004773 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004774
4775 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4776 WaitForEncodedFrame(CurrentTimeMs());
4777 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4778 VideoLayersAllocation last_layer_allocation =
4779 sink_.GetLastVideoLayersAllocation();
4780
4781 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4782 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4783 .target_bitrate_per_temporal_layer,
4784 SizeIs(2));
4785 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4786
4787 video_stream_encoder_->Stop();
4788}
4789
4790TEST_F(VideoStreamEncoderTest,
4791 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4792 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4793 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004794 VideoEncoderConfig video_encoder_config;
4795 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4796 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004797 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004798 video_encoder_config.content_type =
4799 VideoEncoderConfig::ContentType::kRealtimeVideo;
4800 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4801 vp9_settings.numberOfSpatialLayers = 2;
4802 vp9_settings.numberOfTemporalLayers = 2;
4803 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4804 vp9_settings.automaticResizeOn = false;
4805 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004806 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004807 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004808 ConfigureEncoder(std::move(video_encoder_config),
4809 VideoStreamEncoder::BitrateAllocationCallbackType::
4810 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004811
4812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004813 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004814
4815 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4816 WaitForEncodedFrame(CurrentTimeMs());
4817 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4818 VideoLayersAllocation last_layer_allocation =
4819 sink_.GetLastVideoLayersAllocation();
4820
4821 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4822 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4823 .target_bitrate_per_temporal_layer,
4824 SizeIs(2));
4825 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4826 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4827 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4828 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4829 .target_bitrate_per_temporal_layer,
4830 SizeIs(2));
4831 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4832 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4833 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4834
4835 // Since full SVC is used, expect the top layer to utilize the full target
4836 // rate.
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4838 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004839 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004840 video_stream_encoder_->Stop();
4841}
4842
4843TEST_F(VideoStreamEncoderTest,
4844 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4845 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4846 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004847 VideoEncoderConfig video_encoder_config;
4848 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4849 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004850 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851 video_encoder_config.content_type =
4852 VideoEncoderConfig::ContentType::kRealtimeVideo;
4853 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4854 vp9_settings.numberOfSpatialLayers = 2;
4855 vp9_settings.numberOfTemporalLayers = 2;
4856 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4857 vp9_settings.automaticResizeOn = false;
4858 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004859 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004860 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004861 ConfigureEncoder(std::move(video_encoder_config),
4862 VideoStreamEncoder::BitrateAllocationCallbackType::
4863 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004864
4865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004866 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004867
4868 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4869 WaitForEncodedFrame(CurrentTimeMs());
4870 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4871 VideoLayersAllocation last_layer_allocation =
4872 sink_.GetLastVideoLayersAllocation();
4873
4874 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4875 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4876 .target_bitrate_per_temporal_layer,
4877 SizeIs(1));
4878 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4879 .target_bitrate_per_temporal_layer,
4880 SizeIs(1));
4881 // Since full SVC is used, expect the top layer to utilize the full target
4882 // rate.
4883 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4884 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004885 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004886 video_stream_encoder_->Stop();
4887}
4888
4889TEST_F(VideoStreamEncoderTest,
4890 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4891 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4892 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004893 VideoEncoderConfig video_encoder_config;
4894 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4895 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004896 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004897 video_encoder_config.content_type =
4898 VideoEncoderConfig::ContentType::kRealtimeVideo;
4899 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4900 vp9_settings.numberOfSpatialLayers = 2;
4901 vp9_settings.numberOfTemporalLayers = 2;
4902 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4903 vp9_settings.automaticResizeOn = false;
4904 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004905 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004906 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004907 ConfigureEncoder(std::move(video_encoder_config),
4908 VideoStreamEncoder::BitrateAllocationCallbackType::
4909 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004910
4911 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004912 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004913
4914 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4915 WaitForEncodedFrame(CurrentTimeMs());
4916 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4917 VideoLayersAllocation last_layer_allocation =
4918 sink_.GetLastVideoLayersAllocation();
4919
4920 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4921 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4922 .target_bitrate_per_temporal_layer,
4923 SizeIs(2));
4924 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4925 .target_bitrate_per_temporal_layer,
4926 SizeIs(2));
4927 // Since KSVC is, spatial layers are independend except on key frames.
4928 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4929 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004930 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004931 video_stream_encoder_->Stop();
4932}
4933
4934TEST_F(VideoStreamEncoderTest,
4935 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4936 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4937 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4938 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004939 VideoEncoderConfig video_encoder_config;
4940 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4941 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004942 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004943 video_encoder_config.content_type =
4944 VideoEncoderConfig::ContentType::kRealtimeVideo;
4945 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4946 vp9_settings.numberOfSpatialLayers = 3;
4947 vp9_settings.numberOfTemporalLayers = 2;
4948 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4949 vp9_settings.automaticResizeOn = false;
4950 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004951 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004952 vp9_settings);
4953 // Simulcast layers are used for enabling/disabling streams.
4954 video_encoder_config.simulcast_layers.resize(3);
4955 video_encoder_config.simulcast_layers[0].active = false;
4956 video_encoder_config.simulcast_layers[1].active = true;
4957 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004958 ConfigureEncoder(std::move(video_encoder_config),
4959 VideoStreamEncoder::BitrateAllocationCallbackType::
4960 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004961
4962 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004963 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004964
4965 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4966 WaitForEncodedFrame(CurrentTimeMs());
4967 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4968 VideoLayersAllocation last_layer_allocation =
4969 sink_.GetLastVideoLayersAllocation();
4970
4971 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4972 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4973 .target_bitrate_per_temporal_layer,
4974 SizeIs(2));
4975 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4976 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4977
4978 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4979 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4980 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4981 .target_bitrate_per_temporal_layer,
4982 SizeIs(2));
4983 // Since full SVC is used, expect the top layer to utilize the full target
4984 // rate.
4985 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4986 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004987 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004988 video_stream_encoder_->Stop();
4989}
4990
4991TEST_F(VideoStreamEncoderTest,
4992 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4993 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4994 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4995 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004996 VideoEncoderConfig video_encoder_config;
4997 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4998 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004999 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000 video_encoder_config.content_type =
5001 VideoEncoderConfig::ContentType::kRealtimeVideo;
5002 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5003 vp9_settings.numberOfSpatialLayers = 3;
5004 vp9_settings.numberOfTemporalLayers = 2;
5005 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5006 vp9_settings.automaticResizeOn = false;
5007 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005008 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005009 vp9_settings);
5010 // Simulcast layers are used for enabling/disabling streams.
5011 video_encoder_config.simulcast_layers.resize(3);
5012 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005013 ConfigureEncoder(std::move(video_encoder_config),
5014 VideoStreamEncoder::BitrateAllocationCallbackType::
5015 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005016
5017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005018 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005019
5020 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5021 WaitForEncodedFrame(CurrentTimeMs());
5022 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5023 VideoLayersAllocation last_layer_allocation =
5024 sink_.GetLastVideoLayersAllocation();
5025
5026 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5027 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5028 .target_bitrate_per_temporal_layer,
5029 SizeIs(2));
5030 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5031 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5032
5033 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5034 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5035 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5036 .target_bitrate_per_temporal_layer,
5037 SizeIs(2));
5038 video_stream_encoder_->Stop();
5039}
5040
5041TEST_F(VideoStreamEncoderTest,
5042 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5043 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5044 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5045 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005046 VideoEncoderConfig video_encoder_config;
5047 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5048 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005049 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005050 video_encoder_config.content_type =
5051 VideoEncoderConfig::ContentType::kRealtimeVideo;
5052 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5053 vp9_settings.numberOfSpatialLayers = 3;
5054 vp9_settings.numberOfTemporalLayers = 2;
5055 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5056 vp9_settings.automaticResizeOn = false;
5057 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005058 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005059 vp9_settings);
5060 // Simulcast layers are used for enabling/disabling streams.
5061 video_encoder_config.simulcast_layers.resize(3);
5062 video_encoder_config.simulcast_layers[0].active = false;
5063 video_encoder_config.simulcast_layers[1].active = false;
5064 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005065 ConfigureEncoder(std::move(video_encoder_config),
5066 VideoStreamEncoder::BitrateAllocationCallbackType::
5067 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005068
5069 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005070 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005071
5072 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5073 WaitForEncodedFrame(CurrentTimeMs());
5074 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5075 VideoLayersAllocation last_layer_allocation =
5076 sink_.GetLastVideoLayersAllocation();
5077
5078 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5079 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5080 .target_bitrate_per_temporal_layer,
5081 SizeIs(2));
5082 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5083 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5084 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5085 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005086 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005087 video_stream_encoder_->Stop();
5088}
5089
5090TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5091 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005092 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005093 kVideoLayersAllocation);
5094 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005095 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005096
5097 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5098 WaitForEncodedFrame(CurrentTimeMs());
5099 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5100 VideoLayersAllocation last_layer_allocation =
5101 sink_.GetLastVideoLayersAllocation();
5102
5103 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5104 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5105 .target_bitrate_per_temporal_layer,
5106 SizeIs(1));
5107 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5108 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005109 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005110 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5111 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5112 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5113 video_stream_encoder_->Stop();
5114}
5115
5116TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005117 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5118 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005119 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005120 kVideoLayersAllocation);
5121
5122 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005123 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005124
5125 video_source_.IncomingCapturedFrame(
5126 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5127 WaitForEncodedFrame(CurrentTimeMs());
5128 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5129 VideoLayersAllocation last_layer_allocation =
5130 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005131 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005132 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5133 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5134 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005135 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005136
5137 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005138 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5139 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005140 video_source_.IncomingCapturedFrame(
5141 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5142 WaitForEncodedFrame(CurrentTimeMs());
5143
5144 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5145 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5146 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5147 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5148 .target_bitrate_per_temporal_layer[0],
5149 DataRate::Zero());
5150
5151 video_stream_encoder_->Stop();
5152}
5153
Per Kjellander4190ce92020-12-15 17:24:55 +01005154TEST_F(VideoStreamEncoderTest,
5155 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5156 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005157 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005158 kVideoLayersAllocation);
5159
5160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005161 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5162 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005163
5164 video_source_.IncomingCapturedFrame(
5165 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5166 WaitForEncodedFrame(CurrentTimeMs());
5167 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5168 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5169 SizeIs(2));
5170 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5171 codec_width_);
5172 EXPECT_EQ(
5173 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5174 codec_height_);
5175
5176 video_source_.IncomingCapturedFrame(
5177 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5178 WaitForEncodedFrame(CurrentTimeMs());
5179 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5180 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5181 SizeIs(2));
5182 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5183 codec_width_ / 2);
5184 EXPECT_EQ(
5185 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5186 codec_height_ / 2);
5187
5188 video_stream_encoder_->Stop();
5189}
5190
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005191TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5192 // 2 TLs configured, temporal layers supported by encoder.
5193 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005194 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005195 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005196 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005197 fake_encoder_.SetTemporalLayersSupported(0, true);
5198
5199 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005200 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005201 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005202 kNumTemporalLayers, /*temporal_id*/ 0,
5203 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005204 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005205 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005206 kNumTemporalLayers, /*temporal_id*/ 1,
5207 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005208 VideoBitrateAllocation expected_bitrate;
5209 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5210 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5211
5212 VerifyAllocatedBitrate(expected_bitrate);
5213 video_stream_encoder_->Stop();
5214}
5215
5216TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5217 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005218 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005219 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005220 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005221 fake_encoder_.SetTemporalLayersSupported(0, false);
5222
5223 // Temporal layers not supported by the encoder.
5224 // Total bitrate should be at ti:0.
5225 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005226 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005227
5228 VerifyAllocatedBitrate(expected_bitrate);
5229 video_stream_encoder_->Stop();
5230}
5231
5232TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005233 webrtc::test::ScopedKeyValueConfig field_trials(
5234 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005235 "WebRTC-Video-QualityScalerSettings/"
5236 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5237 // Reset encoder for field trials to take effect.
5238 ConfigureEncoder(video_encoder_config_.Copy());
5239
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005240 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005241 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005242 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005243 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005244 fake_encoder_.SetTemporalLayersSupported(0, true);
5245 fake_encoder_.SetTemporalLayersSupported(1, false);
5246
5247 const int kS0Bps = 150000;
5248 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005249 kS0Bps *
5250 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5251 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005252 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005253 kS0Bps *
5254 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5255 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005256 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005257 // Temporal layers not supported by si:1.
5258 VideoBitrateAllocation expected_bitrate;
5259 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5260 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5261 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5262
5263 VerifyAllocatedBitrate(expected_bitrate);
5264 video_stream_encoder_->Stop();
5265}
5266
Niels Möller7dc26b72017-12-06 10:27:48 +01005267TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5268 const int kFrameWidth = 1280;
5269 const int kFrameHeight = 720;
5270 const int kFramerate = 24;
5271
Henrik Boström381d1092020-05-12 18:49:07 +02005272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005273 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005274 test::FrameForwarder source;
5275 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005276 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005277
5278 // Insert a single frame, triggering initial configuration.
5279 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5280 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5281
5282 EXPECT_EQ(
5283 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5284 kDefaultFramerate);
5285
5286 // Trigger reconfigure encoder (without resetting the entire instance).
5287 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005288 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5289 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005290 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005291 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005292 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005293 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5294
5295 // Detector should be updated with fps limit from codec config.
5296 EXPECT_EQ(
5297 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5298 kFramerate);
5299
5300 // Trigger overuse, max framerate should be reduced.
5301 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5302 stats.input_frame_rate = kFramerate;
5303 stats_proxy_->SetMockStats(stats);
5304 video_stream_encoder_->TriggerCpuOveruse();
5305 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5306 int adapted_framerate =
5307 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5308 EXPECT_LT(adapted_framerate, kFramerate);
5309
5310 // Trigger underuse, max framerate should go back to codec configured fps.
5311 // Set extra low fps, to make sure it's actually reset, not just incremented.
5312 stats = stats_proxy_->GetStats();
5313 stats.input_frame_rate = adapted_framerate / 2;
5314 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005315 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005316 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5317 EXPECT_EQ(
5318 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5319 kFramerate);
5320
5321 video_stream_encoder_->Stop();
5322}
5323
5324TEST_F(VideoStreamEncoderTest,
5325 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5326 const int kFrameWidth = 1280;
5327 const int kFrameHeight = 720;
5328 const int kLowFramerate = 15;
5329 const int kHighFramerate = 25;
5330
Henrik Boström381d1092020-05-12 18:49:07 +02005331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005332 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005333 test::FrameForwarder source;
5334 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005335 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005336
5337 // Trigger initial configuration.
5338 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005339 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5340 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005341 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005342 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005343 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005344 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005345 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5346
5347 EXPECT_EQ(
5348 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5349 kLowFramerate);
5350
5351 // Trigger overuse, max framerate should be reduced.
5352 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5353 stats.input_frame_rate = kLowFramerate;
5354 stats_proxy_->SetMockStats(stats);
5355 video_stream_encoder_->TriggerCpuOveruse();
5356 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5357 int adapted_framerate =
5358 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5359 EXPECT_LT(adapted_framerate, kLowFramerate);
5360
5361 // Reconfigure the encoder with a new (higher max framerate), max fps should
5362 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005363 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005364 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5365 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005366 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005367 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5368
5369 EXPECT_EQ(
5370 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5371 adapted_framerate);
5372
5373 // Trigger underuse, max framerate should go back to codec configured fps.
5374 stats = stats_proxy_->GetStats();
5375 stats.input_frame_rate = adapted_framerate;
5376 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005377 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005378 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5379 EXPECT_EQ(
5380 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5381 kHighFramerate);
5382
5383 video_stream_encoder_->Stop();
5384}
5385
mflodmancc3d4422017-08-03 08:27:51 -07005386TEST_F(VideoStreamEncoderTest,
5387 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005388 const int kFrameWidth = 1280;
5389 const int kFrameHeight = 720;
5390 const int kFramerate = 24;
5391
Henrik Boström381d1092020-05-12 18:49:07 +02005392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005393 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005394 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005395 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005396 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005397
5398 // Trigger initial configuration.
5399 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005400 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5401 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005402 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005403 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005404 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005405 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005406 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005407
Niels Möller7dc26b72017-12-06 10:27:48 +01005408 EXPECT_EQ(
5409 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5410 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005411
5412 // Trigger overuse, max framerate should be reduced.
5413 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5414 stats.input_frame_rate = kFramerate;
5415 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005416 video_stream_encoder_->TriggerCpuOveruse();
5417 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005418 int adapted_framerate =
5419 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005420 EXPECT_LT(adapted_framerate, kFramerate);
5421
5422 // Change degradation preference to not enable framerate scaling. Target
5423 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005424 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005425 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005426 EXPECT_EQ(
5427 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5428 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005429
mflodmancc3d4422017-08-03 08:27:51 -07005430 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005431}
5432
mflodmancc3d4422017-08-03 08:27:51 -07005433TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005434 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005436 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5437 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5438 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005439 const int kWidth = 640;
5440 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005441
asaperssonfab67072017-04-04 05:51:49 -07005442 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005443
5444 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005445 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005446
5447 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005448 EXPECT_TRUE_WAIT(
5449 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005450
sprangc5d62e22017-04-02 23:53:04 -07005451 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005452
asaperssonfab67072017-04-04 05:51:49 -07005453 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005454 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005455 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005456
5457 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005458 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005459
Henrik Boström2671dac2020-05-19 16:29:09 +02005460 EXPECT_TRUE_WAIT(
5461 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005462
mflodmancc3d4422017-08-03 08:27:51 -07005463 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005464}
5465
mflodmancc3d4422017-08-03 08:27:51 -07005466TEST_F(VideoStreamEncoderTest,
5467 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005468 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005470 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5471 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5472 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005473 const int kWidth = 640;
5474 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005475
5476 // We expect the n initial frames to get dropped.
5477 int i;
5478 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005479 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005480 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005481 }
5482 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005483 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005484 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005485
5486 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005487 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005488
mflodmancc3d4422017-08-03 08:27:51 -07005489 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005490}
5491
mflodmancc3d4422017-08-03 08:27:51 -07005492TEST_F(VideoStreamEncoderTest,
5493 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005494 const int kWidth = 640;
5495 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005496 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005497 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005498
5499 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005500 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005501 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005502
asaperssonfab67072017-04-04 05:51:49 -07005503 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005504 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005505 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005506
mflodmancc3d4422017-08-03 08:27:51 -07005507 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005508}
5509
mflodmancc3d4422017-08-03 08:27:51 -07005510TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005511 const int kWidth = 640;
5512 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005513 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005514
5515 VideoEncoderConfig video_encoder_config;
5516 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5517 // Make format different, to force recreation of encoder.
5518 video_encoder_config.video_format.parameters["foo"] = "foo";
5519 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005520 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005521 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005522 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005523
kthelgasonb83797b2017-02-14 11:57:25 -08005524 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005525 video_stream_encoder_->SetSource(&video_source_,
5526 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005527
asaperssonfab67072017-04-04 05:51:49 -07005528 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005529 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005530 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005531
mflodmancc3d4422017-08-03 08:27:51 -07005532 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005533 fake_encoder_.SetQualityScaling(true);
5534}
5535
Åsa Persson139f4dc2019-08-02 09:29:58 +02005536TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005537 webrtc::test::ScopedKeyValueConfig field_trials(
5538 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005539 "WebRTC-Video-QualityScalerSettings/"
5540 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5541 // Reset encoder for field trials to take effect.
5542 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005543 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5544 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005545 const int kWidth = 640;
5546 const int kHeight = 360;
5547
Henrik Boström381d1092020-05-12 18:49:07 +02005548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005549 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005550 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5551 // Frame should not be dropped.
5552 WaitForEncodedFrame(1);
5553
Henrik Boström381d1092020-05-12 18:49:07 +02005554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005555 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5556 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5557 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005558 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5559 // Frame should not be dropped.
5560 WaitForEncodedFrame(2);
5561
Henrik Boström381d1092020-05-12 18:49:07 +02005562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005563 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5564 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5565 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005566 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5567 // Expect to drop this frame, the wait should time out.
5568 ExpectDroppedFrame();
5569
5570 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005571 EXPECT_TRUE_WAIT(
5572 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005573 video_stream_encoder_->Stop();
5574}
5575
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005576TEST_F(VideoStreamEncoderTest,
5577 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005578 webrtc::test::ScopedKeyValueConfig field_trials(
5579 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005580 "WebRTC-Video-QualityScalerSettings/"
5581 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5582 fake_encoder_.SetQualityScaling(false);
5583 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005584 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5585 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005586 const int kWidth = 640;
5587 const int kHeight = 360;
5588
5589 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005590 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005591 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5592 // Frame should not be dropped.
5593 WaitForEncodedFrame(1);
5594
5595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5596 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5597 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5598 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5599 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5600 // Frame should not be dropped.
5601 WaitForEncodedFrame(2);
5602
5603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5604 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5605 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5606 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5607 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5608 // Not dropped since quality scaling is disabled.
5609 WaitForEncodedFrame(3);
5610
5611 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005612 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005613 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5614
5615 video_stream_encoder_->Stop();
5616}
5617
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005618TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005619 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005620 // Set simulcast.
5621 ResetEncoder("VP8", 3, 1, 1, false);
5622 fake_encoder_.SetQualityScaling(true);
5623 const int kWidth = 1280;
5624 const int kHeight = 720;
5625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005626 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005627 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5628 // Frame should not be dropped.
5629 WaitForEncodedFrame(1);
5630
5631 // Trigger QVGA "singlecast"
5632 // Update the config.
5633 VideoEncoderConfig video_encoder_config;
5634 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5635 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005636 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005637 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005638 "VP8", /*max qp*/ 56, /*screencast*/ false,
5639 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005640 for (auto& layer : video_encoder_config.simulcast_layers) {
5641 layer.num_temporal_layers = 1;
5642 layer.max_framerate = kDefaultFramerate;
5643 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005644 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005645 video_encoder_config.content_type =
5646 VideoEncoderConfig::ContentType::kRealtimeVideo;
5647
5648 video_encoder_config.simulcast_layers[0].active = true;
5649 video_encoder_config.simulcast_layers[1].active = false;
5650 video_encoder_config.simulcast_layers[2].active = false;
5651
5652 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5653 kMaxPayloadLength);
5654 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5655
5656 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5657 // Frame should not be dropped.
5658 WaitForEncodedFrame(2);
5659
5660 // Trigger HD "singlecast"
5661 video_encoder_config.simulcast_layers[0].active = false;
5662 video_encoder_config.simulcast_layers[1].active = false;
5663 video_encoder_config.simulcast_layers[2].active = true;
5664
5665 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5666 kMaxPayloadLength);
5667 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5668
5669 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5670 // Frame should be dropped because of initial frame drop.
5671 ExpectDroppedFrame();
5672
5673 // Expect the sink_wants to specify a scaled frame.
5674 EXPECT_TRUE_WAIT(
5675 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5676 video_stream_encoder_->Stop();
5677}
5678
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005679TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005680 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005681 // Set simulcast.
5682 ResetEncoder("VP9", 1, 1, 3, false);
5683 fake_encoder_.SetQualityScaling(true);
5684 const int kWidth = 1280;
5685 const int kHeight = 720;
5686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005687 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005688 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5689 // Frame should not be dropped.
5690 WaitForEncodedFrame(1);
5691
5692 // Trigger QVGA "singlecast"
5693 // Update the config.
5694 VideoEncoderConfig video_encoder_config;
5695 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5696 &video_encoder_config);
5697 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5698 vp9_settings.numberOfSpatialLayers = 3;
5699 // Since only one layer is active - automatic resize should be enabled.
5700 vp9_settings.automaticResizeOn = true;
5701 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005702 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005703 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005704 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005705 video_encoder_config.content_type =
5706 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005707 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005708 // which SVC layers are active.
5709 video_encoder_config.simulcast_layers.resize(3);
5710
5711 video_encoder_config.simulcast_layers[0].active = true;
5712 video_encoder_config.simulcast_layers[1].active = false;
5713 video_encoder_config.simulcast_layers[2].active = false;
5714
5715 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5716 kMaxPayloadLength);
5717 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5718
5719 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5720 // Frame should not be dropped.
5721 WaitForEncodedFrame(2);
5722
5723 // Trigger HD "singlecast"
5724 video_encoder_config.simulcast_layers[0].active = false;
5725 video_encoder_config.simulcast_layers[1].active = false;
5726 video_encoder_config.simulcast_layers[2].active = true;
5727
5728 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5729 kMaxPayloadLength);
5730 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5731
5732 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5733 // Frame should be dropped because of initial frame drop.
5734 ExpectDroppedFrame();
5735
5736 // Expect the sink_wants to specify a scaled frame.
5737 EXPECT_TRUE_WAIT(
5738 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5739 video_stream_encoder_->Stop();
5740}
5741
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005742TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005743 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5744 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5745 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5746 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5747 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5748 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5749 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5750 fake_encoder_.SetResolutionBitrateLimits(
5751 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5752
5753 VideoEncoderConfig video_encoder_config;
5754 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5755 &video_encoder_config);
5756 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5757 vp9_settings.numberOfSpatialLayers = 3;
5758 // Since only one layer is active - automatic resize should be enabled.
5759 vp9_settings.automaticResizeOn = true;
5760 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005761 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005762 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005763 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005764 video_encoder_config.content_type =
5765 VideoEncoderConfig::ContentType::kRealtimeVideo;
5766 // Simulcast layers are used to indicate which spatial layers are active.
5767 video_encoder_config.simulcast_layers.resize(3);
5768 video_encoder_config.simulcast_layers[0].active = false;
5769 video_encoder_config.simulcast_layers[1].active = true;
5770 video_encoder_config.simulcast_layers[2].active = false;
5771
5772 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5773 kMaxPayloadLength);
5774 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5775
5776 // The encoder bitrate limits for 360p should be used.
5777 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5778 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005779 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5780 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5781 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5782 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5783 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5784 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005785 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005786 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005787 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005788 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005789
5790 // The encoder bitrate limits for 270p should be used.
5791 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5792 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005793 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5794 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5795 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5796 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5797 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5798 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005799 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005800 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005801 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005802 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005803
5804 video_stream_encoder_->Stop();
5805}
5806
5807TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005808 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5809 VideoEncoderConfig video_encoder_config;
5810 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5811 &video_encoder_config);
5812 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5813 vp9_settings.numberOfSpatialLayers = 3;
5814 // Since only one layer is active - automatic resize should be enabled.
5815 vp9_settings.automaticResizeOn = true;
5816 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005817 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005818 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005819 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005820 video_encoder_config.content_type =
5821 VideoEncoderConfig::ContentType::kRealtimeVideo;
5822 // Simulcast layers are used to indicate which spatial layers are active.
5823 video_encoder_config.simulcast_layers.resize(3);
5824 video_encoder_config.simulcast_layers[0].active = false;
5825 video_encoder_config.simulcast_layers[1].active = true;
5826 video_encoder_config.simulcast_layers[2].active = false;
5827
5828 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5829 kMaxPayloadLength);
5830 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5831
5832 // The default bitrate limits for 360p should be used.
5833 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005834 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5835 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005836 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5837 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005838 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5839 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5840 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5841 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5842 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5843 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005844 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005845 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005846 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005847 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005848
5849 // The default bitrate limits for 270p should be used.
5850 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005851 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5852 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005853 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5854 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005855 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5856 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5857 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5858 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5859 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5860 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005861 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005862 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005863 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005864 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005865
5866 video_stream_encoder_->Stop();
5867}
5868
5869TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005870 webrtc::test::ScopedKeyValueConfig field_trials(
5871 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005872 VideoEncoderConfig video_encoder_config;
5873 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5874 &video_encoder_config);
5875 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5876 vp9_settings.numberOfSpatialLayers = 3;
5877 // Since only one layer is active - automatic resize should be enabled.
5878 vp9_settings.automaticResizeOn = true;
5879 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005880 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005881 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005882 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005883 video_encoder_config.content_type =
5884 VideoEncoderConfig::ContentType::kRealtimeVideo;
5885 // Simulcast layers are used to indicate which spatial layers are active.
5886 video_encoder_config.simulcast_layers.resize(3);
5887 video_encoder_config.simulcast_layers[0].active = false;
5888 video_encoder_config.simulcast_layers[1].active = true;
5889 video_encoder_config.simulcast_layers[2].active = false;
5890
5891 // Reset encoder for field trials to take effect.
5892 ConfigureEncoder(video_encoder_config.Copy());
5893
5894 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5895 kMaxPayloadLength);
5896 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5897
5898 // The default bitrate limits for 360p should not be used.
5899 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005900 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5901 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005902 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5903 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005904 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5905 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5906 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5907 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5908 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5909 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005910 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005911 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005912
5913 video_stream_encoder_->Stop();
5914}
5915
5916TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5917 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5918 /*num_spatial_layers=*/1, /*screenshare=*/false);
5919
5920 // The default singlecast bitrate limits for 720p should not be used.
5921 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005922 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5923 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005924 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5925 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005926 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5927 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5928 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5929 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5930 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5931 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005932 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005933 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005934
5935 video_stream_encoder_->Stop();
5936}
5937
5938TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005939 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5940 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5941 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5942 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5943 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5944 fake_encoder_.SetResolutionBitrateLimits(
5945 {kEncoderLimits180p, kEncoderLimits720p});
5946
5947 VideoEncoderConfig video_encoder_config;
5948 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5949 &video_encoder_config);
5950 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5951 vp9_settings.numberOfSpatialLayers = 3;
5952 // Since only one layer is active - automatic resize should be enabled.
5953 vp9_settings.automaticResizeOn = true;
5954 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005955 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005956 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005957 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005958 video_encoder_config.content_type =
5959 VideoEncoderConfig::ContentType::kRealtimeVideo;
5960 // Simulcast layers are used to indicate which spatial layers are active.
5961 video_encoder_config.simulcast_layers.resize(3);
5962 video_encoder_config.simulcast_layers[0].active = true;
5963 video_encoder_config.simulcast_layers[1].active = false;
5964 video_encoder_config.simulcast_layers[2].active = false;
5965
5966 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5967 kMaxPayloadLength);
5968 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5969
5970 // Limits not applied on lowest stream, limits for 180p should not be used.
5971 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5972 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005973 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5974 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5975 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5976 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5977 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5978 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005979 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005980 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005981 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005982 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005983
5984 video_stream_encoder_->Stop();
5985}
5986
5987TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005988 InitialFrameDropActivatesWhenResolutionIncreases) {
5989 const int kWidth = 640;
5990 const int kHeight = 360;
5991
5992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005993 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005994 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5995 // Frame should not be dropped.
5996 WaitForEncodedFrame(1);
5997
5998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005999 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006000 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6001 // Frame should not be dropped, bitrate not too low for frame.
6002 WaitForEncodedFrame(2);
6003
6004 // Incoming resolution increases.
6005 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6006 // Expect to drop this frame, bitrate too low for frame.
6007 ExpectDroppedFrame();
6008
6009 // Expect the sink_wants to specify a scaled frame.
6010 EXPECT_TRUE_WAIT(
6011 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6012 video_stream_encoder_->Stop();
6013}
6014
6015TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6016 const int kWidth = 640;
6017 const int kHeight = 360;
6018 // So that quality scaling doesn't happen by itself.
6019 fake_encoder_.SetQp(kQpHigh);
6020
6021 AdaptingFrameForwarder source(&time_controller_);
6022 source.set_adaptation_enabled(true);
6023 video_stream_encoder_->SetSource(
6024 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6025
6026 int timestamp = 1;
6027
6028 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006029 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006030 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6031 WaitForEncodedFrame(timestamp);
6032 timestamp += 9000;
6033 // Long pause to disable all first BWE drop logic.
6034 AdvanceTime(TimeDelta::Millis(1000));
6035
6036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006037 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006038 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6039 // Not dropped frame, as initial frame drop is disabled by now.
6040 WaitForEncodedFrame(timestamp);
6041 timestamp += 9000;
6042 AdvanceTime(TimeDelta::Millis(100));
6043
6044 // Quality adaptation down.
6045 video_stream_encoder_->TriggerQualityLow();
6046
6047 // Adaptation has an effect.
6048 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6049 5000);
6050
6051 // Frame isn't dropped as initial frame dropper is disabled.
6052 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6053 WaitForEncodedFrame(timestamp);
6054 timestamp += 9000;
6055 AdvanceTime(TimeDelta::Millis(100));
6056
6057 // Quality adaptation up.
6058 video_stream_encoder_->TriggerQualityHigh();
6059
6060 // Adaptation has an effect.
6061 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6062 5000);
6063
6064 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6065 // Frame should not be dropped, as initial framedropper is off.
6066 WaitForEncodedFrame(timestamp);
6067
6068 video_stream_encoder_->Stop();
6069}
6070
Åsa Persson7f354f82021-02-04 15:52:15 +01006071TEST_F(VideoStreamEncoderTest,
6072 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6073 const int kMinStartBps360p = 222000;
6074 fake_encoder_.SetResolutionBitrateLimits(
6075 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6076 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6077 800000)});
6078
6079 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6080 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6081 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6082 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6083 0, 0, 0);
6084 // Frame should not be dropped, bitrate not too low for frame.
6085 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6086 WaitForEncodedFrame(1);
6087
6088 // Incoming resolution increases, initial frame drop activates.
6089 // Frame should be dropped, link allocation too low for frame.
6090 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6091 ExpectDroppedFrame();
6092
6093 // Expect sink_wants to specify a scaled frame.
6094 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6095 5000);
6096 video_stream_encoder_->Stop();
6097}
6098
6099TEST_F(VideoStreamEncoderTest,
6100 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6101 const int kMinStartBps360p = 222000;
6102 fake_encoder_.SetResolutionBitrateLimits(
6103 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6104 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6105 800000)});
6106
6107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6108 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6109 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6110 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6111 0, 0, 0);
6112 // Frame should not be dropped, bitrate not too low for frame.
6113 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6114 WaitForEncodedFrame(1);
6115
6116 // Incoming resolution increases, initial frame drop activates.
6117 // Frame should be dropped, link allocation not too low for frame.
6118 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6119 WaitForEncodedFrame(2);
6120
6121 video_stream_encoder_->Stop();
6122}
6123
Åsa Perssone644a032019-11-08 15:56:00 +01006124TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006125 webrtc::test::ScopedKeyValueConfig field_trials(
6126 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006127 "WebRTC-Video-QualityRampupSettings/"
6128 "min_pixels:921600,min_duration_ms:2000/");
6129
6130 const int kWidth = 1280;
6131 const int kHeight = 720;
6132 const int kFps = 10;
6133 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006134
6135 // Reset encoder for field trials to take effect.
6136 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006137 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006138 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006139 ConfigureEncoder(std::move(config));
6140 fake_encoder_.SetQp(kQpLow);
6141
6142 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006143 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006144 source.set_adaptation_enabled(true);
6145 video_stream_encoder_->SetSource(&source,
6146 DegradationPreference::MAINTAIN_FRAMERATE);
6147
6148 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006149 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006151 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006152
6153 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006154 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006155 int64_t timestamp_ms = kFrameIntervalMs;
6156 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6157 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006158 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6159 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006160
6161 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6163 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006164
Artem Titovab30d722021-07-27 16:22:11 +02006165 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006166 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006167 for (size_t i = 1; i <= 10; i++) {
6168 timestamp_ms += kFrameIntervalMs;
6169 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6170 WaitForEncodedFrame(timestamp_ms);
6171 }
Åsa Persson06defc42021-09-10 15:28:48 +02006172
6173 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6174 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6175 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6176 timestamp_ms += kFrameIntervalMs;
6177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6178 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006179 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6180 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6181
Åsa Persson06defc42021-09-10 15:28:48 +02006182 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006183 timestamp_ms += kFrameIntervalMs;
6184 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6185 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006186 // The ramp-up code involves the adaptation queue, give it time to execute.
6187 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006188 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006189 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006190
6191 // Frame should not be adapted.
6192 timestamp_ms += kFrameIntervalMs;
6193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6194 WaitForEncodedFrame(kWidth, kHeight);
6195 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6196
6197 video_stream_encoder_->Stop();
6198}
6199
mflodmancc3d4422017-08-03 08:27:51 -07006200TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006201 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006202 webrtc::test::ScopedKeyValueConfig field_trials(
6203 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006204 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006205 source.set_adaptation_enabled(true);
6206 video_stream_encoder_->SetSource(&source,
6207 DegradationPreference::MAINTAIN_FRAMERATE);
6208 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006209 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006210 fake_encoder_.SetQp(kQpHigh + 1);
6211 const int kWidth = 1280;
6212 const int kHeight = 720;
6213 const int64_t kFrameIntervalMs = 100;
6214 int64_t timestamp_ms = kFrameIntervalMs;
6215 for (size_t i = 1; i <= 100; i++) {
6216 timestamp_ms += kFrameIntervalMs;
6217 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6218 WaitForEncodedFrame(timestamp_ms);
6219 }
6220 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6221 // for the first time.
6222 // TODO(eshr): We should avoid these waits by using threads with simulated
6223 // time.
6224 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6225 2000 * 2.5 * 2);
6226 timestamp_ms += kFrameIntervalMs;
6227 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6228 WaitForEncodedFrame(timestamp_ms);
6229 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6230 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6231 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6232
6233 // Disable Quality scaling by turning off scaler on the encoder and
6234 // reconfiguring.
6235 fake_encoder_.SetQualityScaling(false);
6236 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6237 kMaxPayloadLength);
6238 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006239 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006240 // Since we turned off the quality scaler, the adaptations made by it are
6241 // removed.
6242 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6243 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6244
6245 video_stream_encoder_->Stop();
6246}
6247
6248TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006249 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6250 const int kTooSmallWidth = 10;
6251 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006252 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006253 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006254
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006255 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006256 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006257 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006258 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006259 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006260 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6261
6262 // Trigger adapt down, too small frame, expect no change.
6263 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006264 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006265 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006266 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006267 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6268 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6269
mflodmancc3d4422017-08-03 08:27:51 -07006270 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006271}
6272
mflodmancc3d4422017-08-03 08:27:51 -07006273TEST_F(VideoStreamEncoderTest,
6274 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006275 const int kTooSmallWidth = 10;
6276 const int kTooSmallHeight = 10;
6277 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006279 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006280
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006281 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006282 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006283 video_stream_encoder_->SetSource(&source,
6284 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006285 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006286 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6288
6289 // Trigger adapt down, expect limited framerate.
6290 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006291 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006292 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006293 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006294 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6295 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6296 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6297
6298 // Trigger adapt down, too small frame, expect no change.
6299 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006300 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006301 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006302 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006303 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6304 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6305 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6306
mflodmancc3d4422017-08-03 08:27:51 -07006307 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006308}
6309
mflodmancc3d4422017-08-03 08:27:51 -07006310TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006311 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006313 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006314 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006315 const int kFrameWidth = 1280;
6316 const int kFrameHeight = 720;
6317 video_source_.IncomingCapturedFrame(
6318 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006319 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006320 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006321}
6322
sprangb1ca0732017-02-01 08:38:12 -08006323// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006324TEST_F(VideoStreamEncoderTest,
6325 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006327 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006328
6329 const int kFrameWidth = 1280;
6330 const int kFrameHeight = 720;
6331 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006332 // requested by
6333 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006334 video_source_.set_adaptation_enabled(true);
6335
6336 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006337 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006338 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006339
6340 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006341 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006342 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006343 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006344 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006345
asaperssonfab67072017-04-04 05:51:49 -07006346 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006347 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006348 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006349 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006350 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006351
mflodmancc3d4422017-08-03 08:27:51 -07006352 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006353}
sprangfe627f32017-03-29 08:24:59 -07006354
mflodmancc3d4422017-08-03 08:27:51 -07006355TEST_F(VideoStreamEncoderTest,
6356 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006357 const int kFrameWidth = 1280;
6358 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006359
Henrik Boström381d1092020-05-12 18:49:07 +02006360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006363 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006364 video_source_.set_adaptation_enabled(true);
6365
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006366 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006367
6368 video_source_.IncomingCapturedFrame(
6369 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006370 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006371
6372 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006373 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006374
6375 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006376 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006377 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006378 video_source_.IncomingCapturedFrame(
6379 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006380 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006381 }
6382
6383 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006385 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006386 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006390 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006391 ++num_frames_dropped;
6392 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006393 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006394 }
6395 }
6396
sprang4847ae62017-06-27 07:06:52 -07006397 // Add some slack to account for frames dropped by the frame dropper.
6398 const int kErrorMargin = 1;
6399 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006400 kErrorMargin);
6401
6402 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006403 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006404 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006405 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006406 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006407 video_source_.IncomingCapturedFrame(
6408 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006409 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006410 ++num_frames_dropped;
6411 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006412 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006413 }
6414 }
sprang4847ae62017-06-27 07:06:52 -07006415 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006416 kErrorMargin);
6417
6418 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006419 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006420 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006421 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006422 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006423 video_source_.IncomingCapturedFrame(
6424 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006425 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006426 ++num_frames_dropped;
6427 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006428 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006429 }
6430 }
sprang4847ae62017-06-27 07:06:52 -07006431 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006432 kErrorMargin);
6433
6434 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006435 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006436 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006437 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006438 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006439 video_source_.IncomingCapturedFrame(
6440 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006441 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006442 ++num_frames_dropped;
6443 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006444 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006445 }
6446 }
6447 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6448
mflodmancc3d4422017-08-03 08:27:51 -07006449 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006450}
6451
mflodmancc3d4422017-08-03 08:27:51 -07006452TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006453 const int kFramerateFps = 5;
6454 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006455 const int kFrameWidth = 1280;
6456 const int kFrameHeight = 720;
6457
sprang4847ae62017-06-27 07:06:52 -07006458 // Reconfigure encoder with two temporal layers and screensharing, which will
6459 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006460 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006461
Henrik Boström381d1092020-05-12 18:49:07 +02006462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006463 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006464 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006465 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006466 video_source_.set_adaptation_enabled(true);
6467
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006468 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006469
6470 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006471 rtc::VideoSinkWants last_wants;
6472 do {
6473 last_wants = video_source_.sink_wants();
6474
sprangc5d62e22017-04-02 23:53:04 -07006475 // Insert frames to get a new fps estimate...
6476 for (int j = 0; j < kFramerateFps; ++j) {
6477 video_source_.IncomingCapturedFrame(
6478 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006479 if (video_source_.last_sent_width()) {
6480 sink_.WaitForEncodedFrame(timestamp_ms);
6481 }
sprangc5d62e22017-04-02 23:53:04 -07006482 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006483 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006484 }
6485 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006486 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006487 } while (video_source_.sink_wants().max_framerate_fps <
6488 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006489
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006490 EXPECT_THAT(video_source_.sink_wants(),
6491 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006492
mflodmancc3d4422017-08-03 08:27:51 -07006493 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006494}
asaperssonf7e294d2017-06-13 23:25:22 -07006495
mflodmancc3d4422017-08-03 08:27:51 -07006496TEST_F(VideoStreamEncoderTest,
6497 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006498 const int kWidth = 1280;
6499 const int kHeight = 720;
6500 const int64_t kFrameIntervalMs = 150;
6501 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006503 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006504
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006505 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006506 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006507 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006508 video_stream_encoder_->SetSource(&source,
6509 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006510 timestamp_ms += kFrameIntervalMs;
6511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006512 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006513 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6515 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6516 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6517
6518 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006519 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006520 timestamp_ms += kFrameIntervalMs;
6521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006522 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006523 EXPECT_THAT(source.sink_wants(),
6524 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6527 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6528
6529 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006530 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006531 timestamp_ms += kFrameIntervalMs;
6532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006533 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006534 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6537 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6538
6539 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006540 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006541 timestamp_ms += kFrameIntervalMs;
6542 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006543 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006544 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006545 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6547 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6548
6549 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006550 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006551 timestamp_ms += kFrameIntervalMs;
6552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006553 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006554 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6557 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6558
6559 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006560 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006561 timestamp_ms += kFrameIntervalMs;
6562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006564 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6567 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6568
6569 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006570 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006571 timestamp_ms += kFrameIntervalMs;
6572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006573 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006574 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6577 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6578
6579 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006580 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006581 timestamp_ms += kFrameIntervalMs;
6582 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006583 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006584 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006585 rtc::VideoSinkWants last_wants = source.sink_wants();
6586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6588 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6589
6590 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006591 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006594 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006595 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6598 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6599
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006600 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006601 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006602 timestamp_ms += kFrameIntervalMs;
6603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006604 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006605 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006606 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
6610 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006612 timestamp_ms += kFrameIntervalMs;
6613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006615 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6618 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6619
6620 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006621 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006622 timestamp_ms += kFrameIntervalMs;
6623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006624 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006625 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6628 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6629
6630 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006631 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006632 timestamp_ms += kFrameIntervalMs;
6633 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006634 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006635 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6638 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6639
6640 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006641 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006642 timestamp_ms += kFrameIntervalMs;
6643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006644 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006645 EXPECT_THAT(source.sink_wants(), FpsMax());
6646 EXPECT_EQ(source.sink_wants().max_pixel_count,
6647 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6650 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6651
6652 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006653 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006654 timestamp_ms += kFrameIntervalMs;
6655 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006656 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006657 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006658 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6659 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6660 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6661
Åsa Persson30ab0152019-08-27 12:22:33 +02006662 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006663 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006664 timestamp_ms += kFrameIntervalMs;
6665 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006666 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006667 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006668 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006669 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6671 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6672
6673 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006674 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006676 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6677
mflodmancc3d4422017-08-03 08:27:51 -07006678 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006679}
6680
mflodmancc3d4422017-08-03 08:27:51 -07006681TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006682 const int kWidth = 1280;
6683 const int kHeight = 720;
6684 const int64_t kFrameIntervalMs = 150;
6685 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006687 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006688
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006689 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006690 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006691 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006692 video_stream_encoder_->SetSource(&source,
6693 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006694 timestamp_ms += kFrameIntervalMs;
6695 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006696 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006697 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006698 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6699 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6700 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6701 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6703 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6704
6705 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006706 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006707 timestamp_ms += kFrameIntervalMs;
6708 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006709 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006710 EXPECT_THAT(source.sink_wants(),
6711 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006712 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6713 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6714 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6715 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6716 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6718
6719 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006720 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006721 timestamp_ms += kFrameIntervalMs;
6722 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006723 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006724 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006725 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6726 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6727 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6728 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6729 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6731
6732 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006733 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006734 timestamp_ms += kFrameIntervalMs;
6735 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006736 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006737 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006738 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006739 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6740 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6741 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6742 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6743 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6744
Evan Shrubsole64469032020-06-11 10:45:29 +02006745 // Trigger cpu adapt up, expect no change since QP is most limited.
6746 {
6747 // Store current sink wants since we expect no change and if there is no
6748 // change then last_wants() is not updated.
6749 auto previous_sink_wants = source.sink_wants();
6750 video_stream_encoder_->TriggerCpuUnderuse();
6751 timestamp_ms += kFrameIntervalMs;
6752 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6753 WaitForEncodedFrame(timestamp_ms);
6754 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6755 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6756 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6757 }
6758
6759 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6760 video_stream_encoder_->TriggerQualityHigh();
6761 timestamp_ms += kFrameIntervalMs;
6762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6763 WaitForEncodedFrame(timestamp_ms);
6764 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6765 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6766 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6767 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6768 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6769 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6770 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6771
6772 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6773 // expect increased resolution (960x540@30fps).
6774 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006775 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006776 timestamp_ms += kFrameIntervalMs;
6777 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006778 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006779 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006780 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6782 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6784 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006785 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006786
Evan Shrubsole64469032020-06-11 10:45:29 +02006787 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6788 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006789 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006790 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006791 timestamp_ms += kFrameIntervalMs;
6792 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006793 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006794 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6800 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006801 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006802
6803 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006804 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006806 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006807 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006808
mflodmancc3d4422017-08-03 08:27:51 -07006809 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006810}
6811
mflodmancc3d4422017-08-03 08:27:51 -07006812TEST_F(VideoStreamEncoderTest,
6813 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006814 const int kWidth = 640;
6815 const int kHeight = 360;
6816 const int kFpsLimit = 15;
6817 const int64_t kFrameIntervalMs = 150;
6818 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006820 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006821
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006822 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006823 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006824 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006825 video_stream_encoder_->SetSource(&source,
6826 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006827 timestamp_ms += kFrameIntervalMs;
6828 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006829 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006830 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006831 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6832 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6833 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6834 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6835 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6836 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6837
6838 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006839 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006840 timestamp_ms += kFrameIntervalMs;
6841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006842 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006843 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006844 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6845 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6846 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6847 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6848 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6849 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6850
6851 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006852 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006853 timestamp_ms += kFrameIntervalMs;
6854 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006855 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006856 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006857 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006858 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006859 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6860 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6861 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6862 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6863
Evan Shrubsole64469032020-06-11 10:45:29 +02006864 // Trigger cpu adapt up, expect no change because quality is most limited.
6865 {
6866 auto previous_sink_wants = source.sink_wants();
6867 // Store current sink wants since we expect no change ind if there is no
6868 // change then last__wants() is not updated.
6869 video_stream_encoder_->TriggerCpuUnderuse();
6870 timestamp_ms += kFrameIntervalMs;
6871 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6872 WaitForEncodedFrame(timestamp_ms);
6873 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6874 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6875 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6876 }
6877
6878 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6879 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006880 timestamp_ms += kFrameIntervalMs;
6881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006882 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006883 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006884 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6885 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6886 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006887 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6888 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6889 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006890
Evan Shrubsole64469032020-06-11 10:45:29 +02006891 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006892 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006893 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006894 timestamp_ms += kFrameIntervalMs;
6895 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006896 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006897 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6899 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6900 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6901 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6902 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006903 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006904
6905 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006906 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006907 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006908 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006909 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006910
mflodmancc3d4422017-08-03 08:27:51 -07006911 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006912}
6913
mflodmancc3d4422017-08-03 08:27:51 -07006914TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006915 const int kFrameWidth = 1920;
6916 const int kFrameHeight = 1080;
6917 // 3/4 of 1920.
6918 const int kAdaptedFrameWidth = 1440;
6919 // 3/4 of 1080 rounded down to multiple of 4.
6920 const int kAdaptedFrameHeight = 808;
6921 const int kFramerate = 24;
6922
Henrik Boström381d1092020-05-12 18:49:07 +02006923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006924 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006925 // Trigger reconfigure encoder (without resetting the entire instance).
6926 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006927 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6928 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006929 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006930 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006931 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006932 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006933 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006934 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006935
6936 video_source_.set_adaptation_enabled(true);
6937
6938 video_source_.IncomingCapturedFrame(
6939 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006940 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006941
6942 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006943 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006944 video_source_.IncomingCapturedFrame(
6945 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006946 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006947
mflodmancc3d4422017-08-03 08:27:51 -07006948 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006949}
6950
mflodmancc3d4422017-08-03 08:27:51 -07006951TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006952 const int kFrameWidth = 1280;
6953 const int kFrameHeight = 720;
6954 const int kLowFps = 2;
6955 const int kHighFps = 30;
6956
Henrik Boström381d1092020-05-12 18:49:07 +02006957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006958 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006959
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006960 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006961 max_framerate_ = kLowFps;
6962
6963 // Insert 2 seconds of 2fps video.
6964 for (int i = 0; i < kLowFps * 2; ++i) {
6965 video_source_.IncomingCapturedFrame(
6966 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6967 WaitForEncodedFrame(timestamp_ms);
6968 timestamp_ms += 1000 / kLowFps;
6969 }
6970
6971 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006972 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006973 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006974 video_source_.IncomingCapturedFrame(
6975 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6976 WaitForEncodedFrame(timestamp_ms);
6977 timestamp_ms += 1000 / kLowFps;
6978
6979 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6980
6981 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006982 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006983 const int kFrameIntervalMs = 1000 / kHighFps;
6984 max_framerate_ = kHighFps;
6985 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6986 video_source_.IncomingCapturedFrame(
6987 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6988 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6989 // be dropped if the encoder hans't been updated with the new higher target
6990 // framerate yet, causing it to overshoot the target bitrate and then
6991 // suffering the wrath of the media optimizer.
6992 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6993 timestamp_ms += kFrameIntervalMs;
6994 }
6995
6996 // Don expect correct measurement just yet, but it should be higher than
6997 // before.
6998 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6999
mflodmancc3d4422017-08-03 08:27:51 -07007000 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007001}
7002
mflodmancc3d4422017-08-03 08:27:51 -07007003TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007004 const int kFrameWidth = 1280;
7005 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007006 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007007 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007008 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007009
Henrik Boström381d1092020-05-12 18:49:07 +02007010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007011 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007012 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007013
7014 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007015 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007016 video_source_.IncomingCapturedFrame(
7017 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7018 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007019 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007020
7021 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007023 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007024
7025 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007026 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007027 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007028
Per Kjellanderdcef6412020-10-07 15:09:05 +02007029 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007030 video_source_.IncomingCapturedFrame(
7031 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7032 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007033 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007034
mflodmancc3d4422017-08-03 08:27:51 -07007035 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007036}
ilnik6b826ef2017-06-16 06:53:48 -07007037
Niels Möller4db138e2018-04-19 09:04:13 +02007038TEST_F(VideoStreamEncoderTest,
7039 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7040 const int kFrameWidth = 1280;
7041 const int kFrameHeight = 720;
7042 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007044 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007045 video_source_.IncomingCapturedFrame(
7046 CreateFrame(1, kFrameWidth, kFrameHeight));
7047 WaitForEncodedFrame(1);
7048 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7049 .low_encode_usage_threshold_percent,
7050 default_options.low_encode_usage_threshold_percent);
7051 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7052 .high_encode_usage_threshold_percent,
7053 default_options.high_encode_usage_threshold_percent);
7054 video_stream_encoder_->Stop();
7055}
7056
7057TEST_F(VideoStreamEncoderTest,
7058 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7059 const int kFrameWidth = 1280;
7060 const int kFrameHeight = 720;
7061 CpuOveruseOptions hardware_options;
7062 hardware_options.low_encode_usage_threshold_percent = 150;
7063 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007064 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007065
Henrik Boström381d1092020-05-12 18:49:07 +02007066 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007067 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007068 video_source_.IncomingCapturedFrame(
7069 CreateFrame(1, kFrameWidth, kFrameHeight));
7070 WaitForEncodedFrame(1);
7071 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7072 .low_encode_usage_threshold_percent,
7073 hardware_options.low_encode_usage_threshold_percent);
7074 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7075 .high_encode_usage_threshold_percent,
7076 hardware_options.high_encode_usage_threshold_percent);
7077 video_stream_encoder_->Stop();
7078}
7079
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007080TEST_F(VideoStreamEncoderTest,
7081 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7082 const int kFrameWidth = 1280;
7083 const int kFrameHeight = 720;
7084
7085 const CpuOveruseOptions default_options;
7086 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007087 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007088 video_source_.IncomingCapturedFrame(
7089 CreateFrame(1, kFrameWidth, kFrameHeight));
7090 WaitForEncodedFrame(1);
7091 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7092 .low_encode_usage_threshold_percent,
7093 default_options.low_encode_usage_threshold_percent);
7094 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7095 .high_encode_usage_threshold_percent,
7096 default_options.high_encode_usage_threshold_percent);
7097
7098 CpuOveruseOptions hardware_options;
7099 hardware_options.low_encode_usage_threshold_percent = 150;
7100 hardware_options.high_encode_usage_threshold_percent = 200;
7101 fake_encoder_.SetIsHardwareAccelerated(true);
7102
7103 video_source_.IncomingCapturedFrame(
7104 CreateFrame(2, kFrameWidth, kFrameHeight));
7105 WaitForEncodedFrame(2);
7106
7107 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7108 .low_encode_usage_threshold_percent,
7109 hardware_options.low_encode_usage_threshold_percent);
7110 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7111 .high_encode_usage_threshold_percent,
7112 hardware_options.high_encode_usage_threshold_percent);
7113
7114 video_stream_encoder_->Stop();
7115}
7116
Niels Möller6bb5ab92019-01-11 11:11:10 +01007117TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7118 const int kFrameWidth = 320;
7119 const int kFrameHeight = 240;
7120 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007121 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007122 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7123
Henrik Boström381d1092020-05-12 18:49:07 +02007124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007125 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007126
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007127 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007128 max_framerate_ = kFps;
7129
7130 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7131 fake_encoder_.SimulateOvershoot(1.0);
7132 int num_dropped = 0;
7133 for (int i = 0; i < kNumFramesInRun; ++i) {
7134 video_source_.IncomingCapturedFrame(
7135 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7136 // Wait up to two frame durations for a frame to arrive.
7137 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7138 ++num_dropped;
7139 }
7140 timestamp_ms += 1000 / kFps;
7141 }
7142
Erik Språnga8d48ab2019-02-08 14:17:40 +01007143 // Framerate should be measured to be near the expected target rate.
7144 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7145
7146 // Frame drops should be within 5% of expected 0%.
7147 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007148
7149 // Make encoder produce frames at double the expected bitrate during 3 seconds
7150 // of video, verify number of drops. Rate needs to be slightly changed in
7151 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007152 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007153 const RateControlSettings trials =
7154 RateControlSettings::ParseFromFieldTrials();
7155 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007156 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007157 // frame dropping since the adjuter will try to just lower the target
7158 // bitrate rather than drop frames. If network headroom can be used, it
7159 // doesn't push back as hard so we don't need quite as much overshoot.
7160 // These numbers are unfortunately a bit magical but there's not trivial
7161 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007162 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007163 }
7164 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007165 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007166 kTargetBitrate + DataRate::KilobitsPerSec(1),
7167 kTargetBitrate + DataRate::KilobitsPerSec(1),
7168 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007169 num_dropped = 0;
7170 for (int i = 0; i < kNumFramesInRun; ++i) {
7171 video_source_.IncomingCapturedFrame(
7172 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7173 // Wait up to two frame durations for a frame to arrive.
7174 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7175 ++num_dropped;
7176 }
7177 timestamp_ms += 1000 / kFps;
7178 }
7179
Henrik Boström381d1092020-05-12 18:49:07 +02007180 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007181 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007182
7183 // Target framerate should be still be near the expected target, despite
7184 // the frame drops.
7185 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7186
7187 // Frame drops should be within 5% of expected 50%.
7188 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007189
7190 video_stream_encoder_->Stop();
7191}
7192
7193TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7194 const int kFrameWidth = 320;
7195 const int kFrameHeight = 240;
7196 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007197 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007198
7199 ASSERT_GT(max_framerate_, kActualInputFps);
7200
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007201 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007202 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007203 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007204 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007205
7206 // Insert 3 seconds of video, with an input fps lower than configured max.
7207 for (int i = 0; i < kActualInputFps * 3; ++i) {
7208 video_source_.IncomingCapturedFrame(
7209 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7210 // Wait up to two frame durations for a frame to arrive.
7211 WaitForEncodedFrame(timestamp_ms);
7212 timestamp_ms += 1000 / kActualInputFps;
7213 }
7214
7215 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7216
7217 video_stream_encoder_->Stop();
7218}
7219
Markus Handell9a478b52021-11-18 16:07:01 +01007220TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007221 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007222 test::FrameForwarder source;
7223 video_stream_encoder_->SetSource(&source,
7224 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007225 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007226 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007227
Markus Handell9a478b52021-11-18 16:07:01 +01007228 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007229 WaitForEncodedFrame(1);
7230 // On the very first frame full update should be forced.
7231 rect = fake_encoder_.GetLastUpdateRect();
7232 EXPECT_EQ(rect.offset_x, 0);
7233 EXPECT_EQ(rect.offset_y, 0);
7234 EXPECT_EQ(rect.height, codec_height_);
7235 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007236 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7237 // scheduled for processing during encoder queue processing of frame 2.
7238 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7239 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007240 WaitForEncodedFrame(3);
7241 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7242 rect = fake_encoder_.GetLastUpdateRect();
7243 EXPECT_EQ(rect.offset_x, 1);
7244 EXPECT_EQ(rect.offset_y, 0);
7245 EXPECT_EQ(rect.width, 10);
7246 EXPECT_EQ(rect.height, 1);
7247
Markus Handell9a478b52021-11-18 16:07:01 +01007248 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007249 WaitForEncodedFrame(4);
7250 // Previous frame was encoded, so no accumulation should happen.
7251 rect = fake_encoder_.GetLastUpdateRect();
7252 EXPECT_EQ(rect.offset_x, 0);
7253 EXPECT_EQ(rect.offset_y, 0);
7254 EXPECT_EQ(rect.width, 1);
7255 EXPECT_EQ(rect.height, 1);
7256
7257 video_stream_encoder_->Stop();
7258}
7259
Erik Språngd7329ca2019-02-21 21:19:53 +01007260TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007261 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007262 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007263
7264 // First frame is always keyframe.
7265 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7266 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007267 EXPECT_THAT(
7268 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007269 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007270
7271 // Insert delta frame.
7272 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7273 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007274 EXPECT_THAT(
7275 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007276 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007277
7278 // Request next frame be a key-frame.
7279 video_stream_encoder_->SendKeyFrame();
7280 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7281 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007282 EXPECT_THAT(
7283 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007284 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007285
7286 video_stream_encoder_->Stop();
7287}
7288
7289TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7290 // Setup simulcast with three streams.
7291 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007293 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7294 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007295 // Wait for all three layers before triggering event.
7296 sink_.SetNumExpectedLayers(3);
7297
7298 // First frame is always keyframe.
7299 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7300 WaitForEncodedFrame(1);
7301 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007302 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7303 VideoFrameType::kVideoFrameKey,
7304 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007305
7306 // Insert delta frame.
7307 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7308 WaitForEncodedFrame(2);
7309 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007310 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7311 VideoFrameType::kVideoFrameDelta,
7312 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007313
7314 // Request next frame be a key-frame.
7315 // Only first stream is configured to produce key-frame.
7316 video_stream_encoder_->SendKeyFrame();
7317 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7318 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007319
7320 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7321 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007322 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007323 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007324 VideoFrameType::kVideoFrameKey,
7325 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007326
7327 video_stream_encoder_->Stop();
7328}
7329
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007330TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007331 // SPS contains VUI with restrictions on the maximum number of reordered
7332 // pictures, there is no need to rewrite the bitstream to enable faster
7333 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007334 ResetEncoder("H264", 1, 1, 1, false);
7335
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007337 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007338 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007339
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007340 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007341 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007342
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007343 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7344 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007345
7346 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007347 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007348
7349 video_stream_encoder_->Stop();
7350}
7351
7352TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007353 // SPS does not contain VUI, the bitstream is will be rewritten with added
7354 // VUI with restrictions on the maximum number of reordered pictures to
7355 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007356 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7357 0x00, 0x00, 0x03, 0x03, 0xF4,
7358 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007359 ResetEncoder("H264", 1, 1, 1, false);
7360
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007362 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007363 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007364
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007365 fake_encoder_.SetEncodedImageData(
7366 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007367
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007368 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7369 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007370
7371 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007372 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007373
7374 video_stream_encoder_->Stop();
7375}
7376
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007377TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7378 const int kFrameWidth = 1280;
7379 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007380 const DataRate kTargetBitrate =
7381 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007382
Henrik Boström381d1092020-05-12 18:49:07 +02007383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007384 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007385 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7386
7387 // Insert a first video frame. It should be dropped because of downscale in
7388 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007389 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007390 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7391 frame.set_rotation(kVideoRotation_270);
7392 video_source_.IncomingCapturedFrame(frame);
7393
7394 ExpectDroppedFrame();
7395
7396 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007397 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007398 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7399 frame.set_rotation(kVideoRotation_90);
7400 video_source_.IncomingCapturedFrame(frame);
7401
7402 WaitForEncodedFrame(timestamp_ms);
7403 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7404
7405 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007406 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007407 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7408 frame.set_rotation(kVideoRotation_180);
7409 video_source_.IncomingCapturedFrame(frame);
7410
7411 WaitForEncodedFrame(timestamp_ms);
7412 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7413
7414 video_stream_encoder_->Stop();
7415}
7416
Erik Språng5056af02019-09-02 15:53:11 +02007417TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7418 const int kFrameWidth = 320;
7419 const int kFrameHeight = 180;
7420
7421 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007423 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7424 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7425 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007426 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007427 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007428 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007429
7430 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007431 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007432 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7433 frame.set_rotation(kVideoRotation_270);
7434 video_source_.IncomingCapturedFrame(frame);
7435 WaitForEncodedFrame(timestamp_ms);
7436
7437 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007438 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007439 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7440 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007442 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007443 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007444 /*link_allocation=*/target_rate,
7445 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007446 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007447 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007448 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7449
7450 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7451 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7452 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007453 DataRate allocation_sum =
7454 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007455 EXPECT_EQ(min_rate, allocation_sum);
7456 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7457
7458 video_stream_encoder_->Stop();
7459}
7460
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007461TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007463 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007464 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007465 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007466 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7467 WaitForEncodedFrame(1);
7468
7469 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7470 ASSERT_TRUE(prev_rate_settings.has_value());
7471 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7472 kDefaultFramerate);
7473
7474 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7475 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7476 timestamp_ms += 1000 / kDefaultFramerate;
7477 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7478 WaitForEncodedFrame(timestamp_ms);
7479 }
7480 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7481 kDefaultFramerate);
7482 // Capture larger frame to trigger a reconfigure.
7483 codec_height_ *= 2;
7484 codec_width_ *= 2;
7485 timestamp_ms += 1000 / kDefaultFramerate;
7486 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7487 WaitForEncodedFrame(timestamp_ms);
7488
7489 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7490 auto current_rate_settings =
7491 fake_encoder_.GetAndResetLastRateControlSettings();
7492 // Ensure we have actually reconfigured twice
7493 // The rate settings should have been set again even though
7494 // they haven't changed.
7495 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007496 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007497
7498 video_stream_encoder_->Stop();
7499}
7500
philipeld9cc8c02019-09-16 14:53:40 +02007501struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007502 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007503 MOCK_METHOD(void,
7504 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007505 (const webrtc::SdpVideoFormat& format,
7506 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007507 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007508};
7509
philipel9b058032020-02-10 11:30:00 +01007510TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7511 constexpr int kDontCare = 100;
7512 StrictMock<MockEncoderSelector> encoder_selector;
7513 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7514 &fake_encoder_, &encoder_selector);
7515 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7516
7517 // Reset encoder for new configuration to take effect.
7518 ConfigureEncoder(video_encoder_config_.Copy());
7519
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007520 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007521
7522 video_source_.IncomingCapturedFrame(
7523 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007524 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007525 video_stream_encoder_->Stop();
7526
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007527 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007528 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007529 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7530 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007531 video_stream_encoder_.reset();
7532}
7533
7534TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7535 constexpr int kDontCare = 100;
7536
7537 NiceMock<MockEncoderSelector> encoder_selector;
7538 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7539 video_send_config_.encoder_settings.encoder_switch_request_callback =
7540 &switch_callback;
7541 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7542 &fake_encoder_, &encoder_selector);
7543 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7544
7545 // Reset encoder for new configuration to take effect.
7546 ConfigureEncoder(video_encoder_config_.Copy());
7547
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007548 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007549 .WillByDefault(Return(SdpVideoFormat("AV1")));
7550 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007551 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7552 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007553
Henrik Boström381d1092020-05-12 18:49:07 +02007554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007555 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7556 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7557 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007558 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007559 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007560 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007561 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007562
7563 video_stream_encoder_->Stop();
7564}
7565
7566TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7567 constexpr int kSufficientBitrateToNotDrop = 1000;
7568 constexpr int kDontCare = 100;
7569
7570 NiceMock<MockVideoEncoder> video_encoder;
7571 NiceMock<MockEncoderSelector> encoder_selector;
7572 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7573 video_send_config_.encoder_settings.encoder_switch_request_callback =
7574 &switch_callback;
7575 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7576 &video_encoder, &encoder_selector);
7577 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7578
7579 // Reset encoder for new configuration to take effect.
7580 ConfigureEncoder(video_encoder_config_.Copy());
7581
7582 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7583 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7584 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007586 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7587 /*stable_target_bitrate=*/
7588 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7589 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007590 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007591 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007592 /*cwnd_reduce_ratio=*/0);
7593
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007594 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007595 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007596 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007597 .WillByDefault(Return(SdpVideoFormat("AV2")));
7598
7599 rtc::Event encode_attempted;
7600 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007601 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7602 /*allow_default_fallback=*/true))
7603 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007604
7605 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7606 encode_attempted.Wait(3000);
7607
Markus Handell28c71802021-11-08 10:11:55 +01007608 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007609
philipel9b058032020-02-10 11:30:00 +01007610 video_stream_encoder_->Stop();
7611
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007612 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7613 // to it's factory, so in order for the encoder instance in the
7614 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7615 // reset the `video_stream_encoder_` here.
7616 video_stream_encoder_.reset();
7617}
7618
7619TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7620 NiceMock<MockVideoEncoder> video_encoder;
7621 NiceMock<MockEncoderSelector> encoder_selector;
7622 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7623 video_send_config_.encoder_settings.encoder_switch_request_callback =
7624 &switch_callback;
7625 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7626 &video_encoder, &encoder_selector);
7627 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7628
7629 // Reset encoder for new configuration to take effect.
7630 ConfigureEncoder(video_encoder_config_.Copy());
7631
7632 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7633 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7634 /*round_trip_time_ms=*/0,
7635 /*cwnd_reduce_ratio=*/0);
7636 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7637
7638 ON_CALL(video_encoder, InitEncode(_, _))
7639 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7640 ON_CALL(encoder_selector, OnEncoderBroken)
7641 .WillByDefault(Return(SdpVideoFormat("AV2")));
7642
7643 rtc::Event encode_attempted;
7644 EXPECT_CALL(switch_callback,
7645 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7646 /*allow_default_fallback=*/true))
7647 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7648
7649 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7650 encode_attempted.Wait(3000);
7651
7652 AdvanceTime(TimeDelta::Zero());
7653
7654 video_stream_encoder_->Stop();
7655
7656 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7657 // to it's factory, so in order for the encoder instance in the
7658 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7659 // reset the `video_stream_encoder_` here.
7660 video_stream_encoder_.reset();
7661}
7662
7663TEST_F(VideoStreamEncoderTest,
7664 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7665 NiceMock<MockVideoEncoder> video_encoder;
7666 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7667 video_send_config_.encoder_settings.encoder_switch_request_callback =
7668 &switch_callback;
7669 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7670 &video_encoder, /*encoder_selector=*/nullptr);
7671 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7672
7673 // Reset encoder for new configuration to take effect.
7674 ConfigureEncoder(video_encoder_config_.Copy());
7675
7676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7677 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7678 /*round_trip_time_ms=*/0,
7679 /*cwnd_reduce_ratio=*/0);
7680 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7681
7682 ON_CALL(video_encoder, InitEncode(_, _))
7683 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7684
7685 rtc::Event encode_attempted;
7686 EXPECT_CALL(switch_callback,
7687 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7688 /*allow_default_fallback=*/true))
7689 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7690
7691 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7692 encode_attempted.Wait(3000);
7693
7694 AdvanceTime(TimeDelta::Zero());
7695
7696 video_stream_encoder_->Stop();
7697
7698 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007699 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007700 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7701 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007702 video_stream_encoder_.reset();
7703}
7704
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007705TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7706 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7707 // VideoEncoder is passed in encoder_factory, it checks whether
7708 // Codec Switch occurs without a crash.
7709 constexpr int kSufficientBitrateToNotDrop = 1000;
7710 constexpr int kDontCare = 100;
7711
7712 NiceMock<MockEncoderSelector> encoder_selector;
7713 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7714 video_send_config_.encoder_settings.encoder_switch_request_callback =
7715 &switch_callback;
7716 auto encoder_factory =
7717 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7718 /*encoder=*/nullptr, &encoder_selector);
7719 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7720
7721 // Reset encoder for new configuration to take effect.
7722 ConfigureEncoder(video_encoder_config_.Copy());
7723 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7724 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7725 // not fail.
7726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7727 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7728 /*stable_target_bitrate=*/
7729 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7730 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7731 /*fraction_lost=*/0,
7732 /*round_trip_time_ms=*/0,
7733 /*cwnd_reduce_ratio=*/0);
7734 ON_CALL(encoder_selector, OnEncoderBroken)
7735 .WillByDefault(Return(SdpVideoFormat("AV2")));
7736 rtc::Event encode_attempted;
7737 EXPECT_CALL(switch_callback,
7738 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7739 /*allow_default_fallback=*/_))
7740 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7741
7742 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7743 encode_attempted.Wait(3000);
7744
7745 AdvanceTime(TimeDelta::Zero());
7746
7747 video_stream_encoder_->Stop();
7748
7749 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7750 // to it's factory, so in order for the encoder instance in the
7751 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7752 // reset the `video_stream_encoder_` here.
7753 video_stream_encoder_.reset();
7754}
7755
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007756TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007757 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007758 const int kFrameWidth = 320;
7759 const int kFrameHeight = 180;
7760
7761 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007762 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007763 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007764 /*target_bitrate=*/rate,
7765 /*stable_target_bitrate=*/rate,
7766 /*link_allocation=*/rate,
7767 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007768 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007769 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007770
7771 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007772 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007773 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7774 frame.set_rotation(kVideoRotation_270);
7775 video_source_.IncomingCapturedFrame(frame);
7776 WaitForEncodedFrame(timestamp_ms);
7777 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7778
7779 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007780 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007782 /*target_bitrate=*/new_stable_rate,
7783 /*stable_target_bitrate=*/new_stable_rate,
7784 /*link_allocation=*/rate,
7785 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007786 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007787 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007788 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7789 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7790 video_stream_encoder_->Stop();
7791}
7792
7793TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007794 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007795 const int kFrameWidth = 320;
7796 const int kFrameHeight = 180;
7797
7798 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007799 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007801 /*target_bitrate=*/rate,
7802 /*stable_target_bitrate=*/rate,
7803 /*link_allocation=*/rate,
7804 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007805 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007806 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007807
7808 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007809 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007810 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7811 frame.set_rotation(kVideoRotation_270);
7812 video_source_.IncomingCapturedFrame(frame);
7813 WaitForEncodedFrame(timestamp_ms);
7814 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7815
7816 // Set a higher target rate without changing the link_allocation. Should not
7817 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007818 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007820 /*target_bitrate=*/rate,
7821 /*stable_target_bitrate=*/new_stable_rate,
7822 /*link_allocation=*/rate,
7823 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007824 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007825 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007826 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7827 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7828 video_stream_encoder_->Stop();
7829}
7830
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007831TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007832 test::ScopedKeyValueConfig field_trials(
7833 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007834 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7835 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7836 const int kFramerateFps = 30;
7837 const int kWidth = 1920;
7838 const int kHeight = 1080;
7839 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7840 // Works on screenshare mode.
7841 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7842 // We rely on the automatic resolution adaptation, but we handle framerate
7843 // adaptation manually by mocking the stats proxy.
7844 video_source_.set_adaptation_enabled(true);
7845
7846 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007847 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007848 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007849 video_stream_encoder_->SetSource(&video_source_,
7850 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007851 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007852
7853 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7854 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7855
7856 // Pass enough frames with the full update to trigger animation detection.
7857 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007858 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007859 frame.set_ntp_time_ms(timestamp_ms);
7860 frame.set_timestamp_us(timestamp_ms * 1000);
7861 video_source_.IncomingCapturedFrame(frame);
7862 WaitForEncodedFrame(timestamp_ms);
7863 }
7864
7865 // Resolution should be limited.
7866 rtc::VideoSinkWants expected;
7867 expected.max_framerate_fps = kFramerateFps;
7868 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007869 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007870
7871 // Pass one frame with no known update.
7872 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007873 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007874 frame.set_ntp_time_ms(timestamp_ms);
7875 frame.set_timestamp_us(timestamp_ms * 1000);
7876 frame.clear_update_rect();
7877
7878 video_source_.IncomingCapturedFrame(frame);
7879 WaitForEncodedFrame(timestamp_ms);
7880
7881 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007882 EXPECT_THAT(video_source_.sink_wants(),
7883 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007884
7885 video_stream_encoder_->Stop();
7886}
7887
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007888TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7889 const int kWidth = 720; // 540p adapted down.
7890 const int kHeight = 405;
7891 const int kNumFrames = 3;
7892 // Works on screenshare mode.
7893 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7894 /*num_spatial_layers=*/2, /*screenshare=*/true);
7895
7896 video_source_.set_adaptation_enabled(true);
7897
7898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007899 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007900
7901 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7902
7903 // Pass enough frames with the full update to trigger animation detection.
7904 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007905 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007906 frame.set_ntp_time_ms(timestamp_ms);
7907 frame.set_timestamp_us(timestamp_ms * 1000);
7908 video_source_.IncomingCapturedFrame(frame);
7909 WaitForEncodedFrame(timestamp_ms);
7910 }
7911
7912 video_stream_encoder_->Stop();
7913}
7914
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007915TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7916 const float downscale_factors[] = {4.0, 2.0, 1.0};
7917 const int number_layers =
7918 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7919 VideoEncoderConfig config;
7920 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7921 for (int i = 0; i < number_layers; ++i) {
7922 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7923 config.simulcast_layers[i].active = true;
7924 }
7925 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007926 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007927 "VP8", /*max qp*/ 56, /*screencast*/ false,
7928 /*screenshare enabled*/ false);
7929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007930 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7931 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007932
7933 // First initialization.
7934 // Encoder should be initialized. Next frame should be key frame.
7935 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7936 sink_.SetNumExpectedLayers(number_layers);
7937 int64_t timestamp_ms = kFrameIntervalMs;
7938 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7939 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007940 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007941 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7942 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7943 VideoFrameType::kVideoFrameKey,
7944 VideoFrameType::kVideoFrameKey}));
7945
7946 // Disable top layer.
7947 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7948 config.simulcast_layers[number_layers - 1].active = false;
7949 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7950 sink_.SetNumExpectedLayers(number_layers - 1);
7951 timestamp_ms += kFrameIntervalMs;
7952 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7953 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007954 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007955 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7956 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7957 VideoFrameType::kVideoFrameDelta,
7958 VideoFrameType::kVideoFrameDelta}));
7959
7960 // Re-enable top layer.
7961 // Encoder should be re-initialized. Next frame should be key frame.
7962 config.simulcast_layers[number_layers - 1].active = true;
7963 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7964 sink_.SetNumExpectedLayers(number_layers);
7965 timestamp_ms += kFrameIntervalMs;
7966 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7967 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007968 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007969 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7970 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7971 VideoFrameType::kVideoFrameKey,
7972 VideoFrameType::kVideoFrameKey}));
7973
7974 // Top layer max rate change.
7975 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7976 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7977 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7978 sink_.SetNumExpectedLayers(number_layers);
7979 timestamp_ms += kFrameIntervalMs;
7980 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7981 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007982 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007983 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7984 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7985 VideoFrameType::kVideoFrameDelta,
7986 VideoFrameType::kVideoFrameDelta}));
7987
7988 // Top layer resolution change.
7989 // Encoder should be re-initialized. Next frame should be key frame.
7990 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7991 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7992 sink_.SetNumExpectedLayers(number_layers);
7993 timestamp_ms += kFrameIntervalMs;
7994 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7995 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007996 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007997 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7998 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7999 VideoFrameType::kVideoFrameKey,
8000 VideoFrameType::kVideoFrameKey}));
8001 video_stream_encoder_->Stop();
8002}
8003
Henrik Boström1124ed12021-02-25 10:30:39 +01008004TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8005 const int kFrameWidth = 1280;
8006 const int kFrameHeight = 720;
8007
8008 SetUp();
8009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008010 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008011
8012 // Capturing a frame should reconfigure the encoder and expose the encoder
8013 // resolution, which is the same as the input frame.
8014 int64_t timestamp_ms = kFrameIntervalMs;
8015 video_source_.IncomingCapturedFrame(
8016 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8017 WaitForEncodedFrame(timestamp_ms);
8018 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8019 EXPECT_THAT(video_source_.sink_wants().resolutions,
8020 ::testing::ElementsAreArray(
8021 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8022
8023 video_stream_encoder_->Stop();
8024}
8025
8026TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8027 // Pick downscale factors such that we never encode at full resolution - this
8028 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008029 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008030 // encoder should not ask for the frame resolution. This allows video frames
8031 // to have the appearence of one resolution but optimize its internal buffers
8032 // for what is actually encoded.
8033 const size_t kNumSimulcastLayers = 3u;
8034 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8035 const int kFrameWidth = 1280;
8036 const int kFrameHeight = 720;
8037 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8038 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8039 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8040 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8041 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8042 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8043
8044 VideoEncoderConfig config;
8045 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8046 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8047 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8048 config.simulcast_layers[i].active = true;
8049 }
8050 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008051 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008052 "VP8", /*max qp*/ 56, /*screencast*/ false,
8053 /*screenshare enabled*/ false);
8054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008055 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8056 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008057
8058 // Capture a frame with all layers active.
8059 int64_t timestamp_ms = kFrameIntervalMs;
8060 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8061 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8062 video_source_.IncomingCapturedFrame(
8063 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8064 WaitForEncodedFrame(timestamp_ms);
8065 // Expect encoded resolutions to match the expected simulcast layers.
8066 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8067 EXPECT_THAT(
8068 video_source_.sink_wants().resolutions,
8069 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8070
8071 // Capture a frame with one of the layers inactive.
8072 timestamp_ms += kFrameIntervalMs;
8073 config.simulcast_layers[2].active = false;
8074 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8075 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8076 video_source_.IncomingCapturedFrame(
8077 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8078 WaitForEncodedFrame(timestamp_ms);
8079
8080 // Expect encoded resolutions to match the expected simulcast layers.
8081 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8082 EXPECT_THAT(video_source_.sink_wants().resolutions,
8083 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8084
8085 // Capture a frame with all but one layer turned off.
8086 timestamp_ms += kFrameIntervalMs;
8087 config.simulcast_layers[1].active = false;
8088 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8089 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8090 video_source_.IncomingCapturedFrame(
8091 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8092 WaitForEncodedFrame(timestamp_ms);
8093
8094 // Expect encoded resolutions to match the expected simulcast layers.
8095 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8096 EXPECT_THAT(video_source_.sink_wants().resolutions,
8097 ::testing::ElementsAreArray({kLayer0Size}));
8098
8099 video_stream_encoder_->Stop();
8100}
8101
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008102TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008103 ResetEncoder("VP8", 1, 1, 1, false);
8104
Niels Möller8b692902021-06-14 12:04:57 +02008105 // Force encoder reconfig.
8106 video_source_.IncomingCapturedFrame(
8107 CreateFrame(1, codec_width_, codec_height_));
8108 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8109
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008110 // Set QP on encoded frame and pass the frame to encode complete callback.
8111 // Since QP is present QP parsing won't be triggered and the original value
8112 // should be kept.
8113 EncodedImage encoded_image;
8114 encoded_image.qp_ = 123;
8115 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8116 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8117 CodecSpecificInfo codec_info;
8118 codec_info.codecType = kVideoCodecVP8;
8119 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8120 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8121 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8122 video_stream_encoder_->Stop();
8123}
8124
8125TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008126 ResetEncoder("VP8", 1, 1, 1, false);
8127
Niels Möller8b692902021-06-14 12:04:57 +02008128 // Force encoder reconfig.
8129 video_source_.IncomingCapturedFrame(
8130 CreateFrame(1, codec_width_, codec_height_));
8131 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8132
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008133 // Pass an encoded frame without QP to encode complete callback. QP should be
8134 // parsed and set.
8135 EncodedImage encoded_image;
8136 encoded_image.qp_ = -1;
8137 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8138 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8139 CodecSpecificInfo codec_info;
8140 codec_info.codecType = kVideoCodecVP8;
8141 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8142 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8143 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8144 video_stream_encoder_->Stop();
8145}
8146
8147TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008148 webrtc::test::ScopedKeyValueConfig field_trials(
8149 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008150
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008151 ResetEncoder("VP8", 1, 1, 1, false);
8152
Niels Möller8b692902021-06-14 12:04:57 +02008153 // Force encoder reconfig.
8154 video_source_.IncomingCapturedFrame(
8155 CreateFrame(1, codec_width_, codec_height_));
8156 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8157
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008158 EncodedImage encoded_image;
8159 encoded_image.qp_ = -1;
8160 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8161 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8162 CodecSpecificInfo codec_info;
8163 codec_info.codecType = kVideoCodecVP8;
8164 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8165 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8166 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8167 video_stream_encoder_->Stop();
8168}
8169
Sergey Silkind19e3b92021-03-16 10:05:30 +00008170TEST_F(VideoStreamEncoderTest,
8171 QualityScalingNotAllowed_QualityScalingDisabled) {
8172 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8173
8174 // Disable scaling settings in encoder info.
8175 fake_encoder_.SetQualityScaling(false);
8176 // Disable quality scaling in encoder config.
8177 video_encoder_config.is_quality_scaling_allowed = false;
8178 ConfigureEncoder(std::move(video_encoder_config));
8179
8180 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008181 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008182
8183 test::FrameForwarder source;
8184 video_stream_encoder_->SetSource(
8185 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8186 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8187 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8188
8189 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8190 WaitForEncodedFrame(1);
8191 video_stream_encoder_->TriggerQualityLow();
8192 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8193
8194 video_stream_encoder_->Stop();
8195}
8196
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008197TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8198 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8199
8200 // Disable scaling settings in encoder info.
8201 fake_encoder_.SetQualityScaling(false);
8202 // Set QP trusted in encoder info.
8203 fake_encoder_.SetIsQpTrusted(true);
8204 // Enable quality scaling in encoder config.
8205 video_encoder_config.is_quality_scaling_allowed = false;
8206 ConfigureEncoder(std::move(video_encoder_config));
8207
8208 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008209 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008210
8211 test::FrameForwarder source;
8212 video_stream_encoder_->SetSource(
8213 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8214 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8215 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8216
8217 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8218 WaitForEncodedFrame(1);
8219 video_stream_encoder_->TriggerQualityLow();
8220 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8221
8222 video_stream_encoder_->Stop();
8223}
8224
Shuhai Pengf2707702021-09-29 17:19:44 +08008225TEST_F(VideoStreamEncoderTest,
8226 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8227 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8228
8229 // Disable scaling settings in encoder info.
8230 fake_encoder_.SetQualityScaling(false);
8231 // Set QP trusted in encoder info.
8232 fake_encoder_.SetIsQpTrusted(true);
8233 // Enable quality scaling in encoder config.
8234 video_encoder_config.is_quality_scaling_allowed = false;
8235 ConfigureEncoder(std::move(video_encoder_config));
8236
8237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008238 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008239
8240 test::FrameForwarder source;
8241 video_stream_encoder_->SetSource(
8242 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8243 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8245
8246 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8247 WaitForEncodedFrame(1);
8248 video_stream_encoder_->TriggerQualityLow();
8249 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8250
8251 video_stream_encoder_->Stop();
8252}
8253
8254TEST_F(VideoStreamEncoderTest,
8255 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8256 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8257
8258 // Disable scaling settings in encoder info.
8259 fake_encoder_.SetQualityScaling(false);
8260 // Set QP trusted in encoder info.
8261 fake_encoder_.SetIsQpTrusted(false);
8262 // Enable quality scaling in encoder config.
8263 video_encoder_config.is_quality_scaling_allowed = false;
8264 ConfigureEncoder(std::move(video_encoder_config));
8265
8266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008267 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008268
8269 test::FrameForwarder source;
8270 video_stream_encoder_->SetSource(
8271 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8272 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8273 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8274
8275 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8276 WaitForEncodedFrame(1);
8277 video_stream_encoder_->TriggerQualityLow();
8278 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8279
8280 video_stream_encoder_->Stop();
8281}
8282
8283TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8284 // Set QP trusted in encoder info.
8285 fake_encoder_.SetIsQpTrusted(false);
8286
8287 const int MinEncBitrateKbps = 30;
8288 const int MaxEncBitrateKbps = 100;
8289 const int MinStartBitrateKbp = 50;
8290 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8291 /*frame_size_pixels=*/codec_width_ * codec_height_,
8292 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8293 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8294 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8295
8296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008297 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008298
8299 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8300
8301 VideoEncoderConfig video_encoder_config;
8302 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8303 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8304 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8305 MinEncBitrateKbps * 1000;
8306 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8307 kMaxPayloadLength);
8308
8309 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8310 WaitForEncodedFrame(1);
8311 EXPECT_EQ(
8312 MaxEncBitrateKbps,
8313 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8314 EXPECT_EQ(
8315 MinEncBitrateKbps,
8316 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8317
8318 video_stream_encoder_->Stop();
8319}
8320
8321TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8322 // Set QP trusted in encoder info.
8323 fake_encoder_.SetIsQpTrusted(false);
8324
8325 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8326 EncoderInfoSettings::
8327 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8328 codec_width_ * codec_height_,
8329 EncoderInfoSettings::
8330 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8331 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8332
8333 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8334 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8335 const int TargetEncBitrate = MaxEncBitrate;
8336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8337 DataRate::BitsPerSec(TargetEncBitrate),
8338 DataRate::BitsPerSec(TargetEncBitrate),
8339 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8340
8341 VideoEncoderConfig video_encoder_config;
8342 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8343 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8344 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8345 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8346 kMaxPayloadLength);
8347
8348 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8349 WaitForEncodedFrame(1);
8350 EXPECT_EQ(
8351 MaxEncBitrate / 1000,
8352 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8353 EXPECT_EQ(
8354 MinEncBitrate / 1000,
8355 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8356
8357 video_stream_encoder_->Stop();
8358}
8359
Erik Språnge4589cb2022-04-06 16:44:30 +02008360TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8361 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8362 /*num_spatial_layers=*/1,
8363 /*screenshare=*/false, /*allocation_callback_type=*/
8364 VideoStreamEncoder::BitrateAllocationCallbackType::
8365 kVideoBitrateAllocationWhenScreenSharing,
8366 /*num_cores=*/3);
8367
8368 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8369 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8370 video_source_.IncomingCapturedFrame(
8371 CreateFrame(1, /*width=*/320, /*height=*/180));
8372 WaitForEncodedFrame(1);
8373 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8374 VideoCodecComplexity::kComplexityNormal);
8375 video_stream_encoder_->Stop();
8376}
8377
8378TEST_F(VideoStreamEncoderTest,
8379 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8380 webrtc::test::ScopedKeyValueConfig field_trials(
8381 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8382
8383 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8384 /*num_spatial_layers=*/1,
8385 /*screenshare=*/false, /*allocation_callback_type=*/
8386 VideoStreamEncoder::BitrateAllocationCallbackType::
8387 kVideoBitrateAllocationWhenScreenSharing,
8388 /*num_cores=*/2);
8389
8390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8391 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8392 video_source_.IncomingCapturedFrame(
8393 CreateFrame(1, /*width=*/320, /*height=*/180));
8394 WaitForEncodedFrame(1);
8395 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8396 VideoCodecComplexity::kComplexityNormal);
8397 video_stream_encoder_->Stop();
8398}
8399
8400TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8401 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8402 /*num_spatial_layers=*/1,
8403 /*screenshare=*/false, /*allocation_callback_type=*/
8404 VideoStreamEncoder::BitrateAllocationCallbackType::
8405 kVideoBitrateAllocationWhenScreenSharing,
8406 /*num_cores=*/2);
8407
8408 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8409 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8410 video_source_.IncomingCapturedFrame(
8411 CreateFrame(1, /*width=*/320, /*height=*/180));
8412 WaitForEncodedFrame(1);
8413 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8414 VideoCodecComplexity::kComplexityLow);
8415 video_stream_encoder_->Stop();
8416}
8417
Sergey Silkind19e3b92021-03-16 10:05:30 +00008418#if !defined(WEBRTC_IOS)
8419// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8420// disabled by default on iOS.
8421TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8422 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8423
8424 // Disable scaling settings in encoder info.
8425 fake_encoder_.SetQualityScaling(false);
8426 // Enable quality scaling in encoder config.
8427 video_encoder_config.is_quality_scaling_allowed = true;
8428 ConfigureEncoder(std::move(video_encoder_config));
8429
8430 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008431 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008432
8433 test::FrameForwarder source;
8434 video_stream_encoder_->SetSource(
8435 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8436 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8438
8439 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8440 WaitForEncodedFrame(1);
8441 video_stream_encoder_->TriggerQualityLow();
8442 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8443
8444 video_stream_encoder_->Stop();
8445}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008446
8447TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8448 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8449
8450 // Disable scaling settings in encoder info.
8451 fake_encoder_.SetQualityScaling(false);
8452 // Set QP trusted in encoder info.
8453 fake_encoder_.SetIsQpTrusted(true);
8454 // Enable quality scaling in encoder config.
8455 video_encoder_config.is_quality_scaling_allowed = true;
8456 ConfigureEncoder(std::move(video_encoder_config));
8457
8458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008459 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008460
8461 test::FrameForwarder source;
8462 video_stream_encoder_->SetSource(
8463 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8464 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8466
8467 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8468 WaitForEncodedFrame(1);
8469 video_stream_encoder_->TriggerQualityLow();
8470 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8471
8472 video_stream_encoder_->Stop();
8473}
Shuhai Pengf2707702021-09-29 17:19:44 +08008474
8475TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8476 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8477
8478 // Disable scaling settings in encoder info.
8479 fake_encoder_.SetQualityScaling(false);
8480 // Set QP not trusted in encoder info.
8481 fake_encoder_.SetIsQpTrusted(false);
8482 // Enable quality scaling in encoder config.
8483 video_encoder_config.is_quality_scaling_allowed = true;
8484 ConfigureEncoder(std::move(video_encoder_config));
8485
8486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008487 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008488
8489 test::FrameForwarder source;
8490 video_stream_encoder_->SetSource(
8491 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8492 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8493 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8494
8495 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8496 WaitForEncodedFrame(1);
8497 video_stream_encoder_->TriggerQualityLow();
8498 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8499 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8501
8502 video_stream_encoder_->Stop();
8503}
8504
8505TEST_F(VideoStreamEncoderTest,
8506 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8507 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8508
8509 // Disable scaling settings in encoder info.
8510 fake_encoder_.SetQualityScaling(false);
8511 // Set QP trusted in encoder info.
8512 fake_encoder_.SetIsQpTrusted(true);
8513 // Enable quality scaling in encoder config.
8514 video_encoder_config.is_quality_scaling_allowed = true;
8515 ConfigureEncoder(std::move(video_encoder_config));
8516
8517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008518 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008519
8520 test::FrameForwarder source;
8521 video_stream_encoder_->SetSource(
8522 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8523 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8525
8526 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8527 WaitForEncodedFrame(1);
8528 video_stream_encoder_->TriggerQualityLow();
8529 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8530 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8531
8532 video_stream_encoder_->Stop();
8533}
8534
8535TEST_F(VideoStreamEncoderTest,
8536 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8537 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8538
8539 // Disable scaling settings in encoder info.
8540 fake_encoder_.SetQualityScaling(false);
8541 // Set QP trusted in encoder info.
8542 fake_encoder_.SetIsQpTrusted(false);
8543 // Enable quality scaling in encoder config.
8544 video_encoder_config.is_quality_scaling_allowed = true;
8545 ConfigureEncoder(std::move(video_encoder_config));
8546
8547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008548 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008549
8550 test::FrameForwarder source;
8551 video_stream_encoder_->SetSource(
8552 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8553 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8555
8556 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8557 WaitForEncodedFrame(1);
8558 video_stream_encoder_->TriggerQualityLow();
8559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8560
8561 video_stream_encoder_->Stop();
8562}
8563
Erik Språnge4589cb2022-04-06 16:44:30 +02008564#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008565
Henrik Boström56db9ff2021-03-24 09:06:45 +01008566// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8567class VideoStreamEncoderWithRealEncoderTest
8568 : public VideoStreamEncoderTest,
8569 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8570 public:
8571 VideoStreamEncoderWithRealEncoderTest()
8572 : VideoStreamEncoderTest(),
8573 codec_type_(std::get<0>(GetParam())),
8574 allow_i420_conversion_(std::get<1>(GetParam())) {}
8575
8576 void SetUp() override {
8577 VideoStreamEncoderTest::SetUp();
8578 std::unique_ptr<VideoEncoder> encoder;
8579 switch (codec_type_) {
8580 case kVideoCodecVP8:
8581 encoder = VP8Encoder::Create();
8582 break;
8583 case kVideoCodecVP9:
8584 encoder = VP9Encoder::Create();
8585 break;
8586 case kVideoCodecAV1:
philipel95701502022-01-18 18:47:52 +01008587 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008588 break;
8589 case kVideoCodecH264:
8590 encoder =
8591 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8592 break;
8593 case kVideoCodecMultiplex:
8594 mock_encoder_factory_for_multiplex_ =
8595 std::make_unique<MockVideoEncoderFactory>();
8596 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8597 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8598 .WillRepeatedly([] { return VP8Encoder::Create(); });
8599 encoder = std::make_unique<MultiplexEncoderAdapter>(
8600 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8601 false);
8602 break;
8603 default:
Artem Titovd3251962021-11-15 16:57:07 +01008604 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008605 }
8606 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8607 }
8608
8609 void TearDown() override {
8610 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008611 // Ensure `video_stream_encoder_` is destroyed before
8612 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008613 video_stream_encoder_.reset();
8614 VideoStreamEncoderTest::TearDown();
8615 }
8616
8617 protected:
8618 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8619 std::unique_ptr<VideoEncoder> encoder) {
8620 // Configure VSE to use the encoder.
8621 encoder_ = std::move(encoder);
8622 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8623 encoder_.get(), &encoder_selector_);
8624 video_send_config_.encoder_settings.encoder_factory =
8625 encoder_proxy_factory_.get();
8626 VideoEncoderConfig video_encoder_config;
8627 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8628 video_encoder_config_ = video_encoder_config.Copy();
8629 ConfigureEncoder(video_encoder_config_.Copy());
8630
8631 // Set bitrate to ensure frame is not dropped.
8632 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008633 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008634 }
8635
8636 const VideoCodecType codec_type_;
8637 const bool allow_i420_conversion_;
8638 NiceMock<MockEncoderSelector> encoder_selector_;
8639 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8640 std::unique_ptr<VideoEncoder> encoder_;
8641 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8642};
8643
8644TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8645 auto native_i420_frame = test::CreateMappableNativeFrame(
8646 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8647 video_source_.IncomingCapturedFrame(native_i420_frame);
8648 WaitForEncodedFrame(codec_width_, codec_height_);
8649
8650 auto mappable_native_buffer =
8651 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8652 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8653 mappable_native_buffer->GetMappedFramedBuffers();
8654 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8655 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8656 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8657 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8658}
8659
8660TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8661 auto native_nv12_frame = test::CreateMappableNativeFrame(
8662 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8663 video_source_.IncomingCapturedFrame(native_nv12_frame);
8664 WaitForEncodedFrame(codec_width_, codec_height_);
8665
8666 auto mappable_native_buffer =
8667 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8668 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8669 mappable_native_buffer->GetMappedFramedBuffers();
8670 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8671 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8672 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8673 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8674
8675 if (!allow_i420_conversion_) {
8676 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8677 }
8678}
8679
Erik Språng7444b192021-06-02 14:02:13 +02008680TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8681 if (codec_type_ == kVideoCodecMultiplex) {
8682 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8683 return;
8684 }
8685
8686 const size_t kNumSpatialLayers = 3u;
8687 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8688 const int kFrameWidth = 1280;
8689 const int kFrameHeight = 720;
8690 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8691 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8692 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8693 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8694 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8695 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8696
8697 VideoEncoderConfig config;
8698 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8699 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008700 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008701 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8702 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8703 vp9_settings.numberOfTemporalLayers = 3;
8704 vp9_settings.automaticResizeOn = false;
8705 config.encoder_specific_settings =
8706 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8707 vp9_settings);
8708 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8709 /*fps=*/30.0,
8710 /*first_active_layer=*/0,
8711 /*num_spatial_layers=*/3,
8712 /*num_temporal_layers=*/3,
8713 /*is_screenshare=*/false);
8714 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8715 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008716 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008717 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8718 /*fps=*/30.0,
8719 /*first_active_layer=*/0,
8720 /*num_spatial_layers=*/3,
8721 /*num_temporal_layers=*/3,
8722 /*is_screenshare=*/false);
8723 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8724 } else {
8725 // Simulcast for VP8/H264.
8726 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8727 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8728 config.simulcast_layers[i].scale_resolution_down_by =
8729 kDownscaleFactors[i];
8730 config.simulcast_layers[i].active = true;
8731 }
8732 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8733 // Turn off frame dropping to prevent flakiness.
8734 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8735 h264_settings.frameDroppingOn = false;
8736 config.encoder_specific_settings = rtc::make_ref_counted<
8737 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8738 }
8739 }
8740
8741 auto set_layer_active = [&](int layer_idx, bool active) {
8742 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8743 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8744 config.spatial_layers[layer_idx].active = active;
8745 } else {
8746 config.simulcast_layers[layer_idx].active = active;
8747 }
8748 };
8749
8750 config.video_stream_factory =
8751 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8752 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8753 /*screencast*/ false,
8754 /*screenshare enabled*/ false);
8755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008756 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8757 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008758
8759 // Capture a frame with all layers active.
8760 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8761 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8762 int64_t timestamp_ms = kFrameIntervalMs;
8763 video_source_.IncomingCapturedFrame(
8764 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8765
8766 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8767 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8768
8769 // Capture a frame with one of the layers inactive.
8770 set_layer_active(2, false);
8771 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8772 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8773 timestamp_ms += kFrameIntervalMs;
8774 video_source_.IncomingCapturedFrame(
8775 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8776 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8777
8778 // New target bitrates signaled based on lower resolution.
8779 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8781 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8782 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8783
8784 // Re-enable the top layer.
8785 set_layer_active(2, true);
8786 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8787 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8788 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8789
8790 // Bitrate target adjusted back up to enable HD layer...
8791 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8792 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8793 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8794 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8795
8796 // ...then add a new frame.
8797 timestamp_ms += kFrameIntervalMs;
8798 video_source_.IncomingCapturedFrame(
8799 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8800 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8801 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8802
8803 video_stream_encoder_->Stop();
8804}
8805
Henrik Boström56db9ff2021-03-24 09:06:45 +01008806std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8807 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8808 VideoCodecType codec_type = std::get<0>(info.param);
8809 bool allow_i420_conversion = std::get<1>(info.param);
8810 std::string str;
8811 switch (codec_type) {
8812 case kVideoCodecGeneric:
8813 str = "Generic";
8814 break;
8815 case kVideoCodecVP8:
8816 str = "VP8";
8817 break;
8818 case kVideoCodecVP9:
8819 str = "VP9";
8820 break;
8821 case kVideoCodecAV1:
8822 str = "AV1";
8823 break;
8824 case kVideoCodecH264:
8825 str = "H264";
8826 break;
8827 case kVideoCodecMultiplex:
8828 str = "Multiplex";
8829 break;
8830 default:
Artem Titovd3251962021-11-15 16:57:07 +01008831 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008832 }
8833 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8834 return str;
8835}
8836
8837constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8838 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8839constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8840 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8841constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008842 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008843constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8844 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8845#if defined(WEBRTC_USE_H264)
8846constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8847 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8848
8849// The windows compiler does not tolerate #if statements inside the
8850// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8851// and without H264).
8852INSTANTIATE_TEST_SUITE_P(
8853 All,
8854 VideoStreamEncoderWithRealEncoderTest,
8855 ::testing::Values(kVP8DisallowConversion,
8856 kVP9DisallowConversion,
8857 kAV1AllowConversion,
8858 kMultiplexDisallowConversion,
8859 kH264AllowConversion),
8860 TestParametersVideoCodecAndAllowI420ConversionToString);
8861#else
8862INSTANTIATE_TEST_SUITE_P(
8863 All,
8864 VideoStreamEncoderWithRealEncoderTest,
8865 ::testing::Values(kVP8DisallowConversion,
8866 kVP9DisallowConversion,
8867 kAV1AllowConversion,
8868 kMultiplexDisallowConversion),
8869 TestParametersVideoCodecAndAllowI420ConversionToString);
8870#endif
8871
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008872class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8873 protected:
8874 void RunTest(const std::vector<VideoStream>& configs,
8875 const int expected_num_init_encode) {
8876 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008877 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008878 InsertFrameAndWaitForEncoded();
8879 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8880 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008881 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8882 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008883
8884 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8885 ConfigureEncoder(configs[1]);
8886 InsertFrameAndWaitForEncoded();
8887 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8888 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008889 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008890 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008891 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008892
8893 video_stream_encoder_->Stop();
8894 }
8895
8896 void ConfigureEncoder(const VideoStream& stream) {
8897 VideoEncoderConfig config;
8898 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8899 config.max_bitrate_bps = stream.max_bitrate_bps;
8900 config.simulcast_layers[0] = stream;
8901 config.video_stream_factory =
8902 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8903 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8904 /*conference_mode=*/false);
8905 video_stream_encoder_->ConfigureEncoder(std::move(config),
8906 kMaxPayloadLength);
8907 }
8908
8909 void OnBitrateUpdated(DataRate bitrate) {
8910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8911 bitrate, bitrate, bitrate, 0, 0, 0);
8912 }
8913
8914 void InsertFrameAndWaitForEncoded() {
8915 timestamp_ms_ += kFrameIntervalMs;
8916 video_source_.IncomingCapturedFrame(
8917 CreateFrame(timestamp_ms_, kWidth, kHeight));
8918 sink_.WaitForEncodedFrame(timestamp_ms_);
8919 }
8920
8921 void ExpectEqual(const VideoCodec& actual,
8922 const VideoStream& expected) const {
8923 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8924 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8925 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8926 static_cast<unsigned int>(expected.min_bitrate_bps));
8927 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8928 static_cast<unsigned int>(expected.max_bitrate_bps));
8929 EXPECT_EQ(actual.simulcastStream[0].width,
8930 kWidth / expected.scale_resolution_down_by);
8931 EXPECT_EQ(actual.simulcastStream[0].height,
8932 kHeight / expected.scale_resolution_down_by);
8933 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8934 expected.num_temporal_layers);
8935 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8936 }
8937
8938 VideoStream DefaultConfig() const {
8939 VideoStream stream;
8940 stream.max_framerate = 25;
8941 stream.min_bitrate_bps = 35000;
8942 stream.max_bitrate_bps = 900000;
8943 stream.scale_resolution_down_by = 1.0;
8944 stream.num_temporal_layers = 1;
8945 stream.bitrate_priority = 1.0;
8946 stream.scalability_mode = "";
8947 return stream;
8948 }
8949
8950 const int kWidth = 640;
8951 const int kHeight = 360;
8952 int64_t timestamp_ms_ = 0;
8953};
8954
8955TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8956 VideoStream config1 = DefaultConfig();
8957 VideoStream config2 = config1;
8958 config2.max_framerate++;
8959
8960 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8961}
8962
8963TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8964 VideoStream config1 = DefaultConfig();
8965 VideoStream config2 = config1;
8966 config2.min_bitrate_bps += 10000;
8967
8968 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8969}
8970
8971TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8972 VideoStream config1 = DefaultConfig();
8973 VideoStream config2 = config1;
8974 config2.max_bitrate_bps += 100000;
8975
8976 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8977}
8978
8979TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8980 VideoStream config1 = DefaultConfig();
8981 VideoStream config2 = config1;
8982 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8983
8984 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8985}
8986
8987TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8988 VideoStream config1 = DefaultConfig();
8989 VideoStream config2 = config1;
8990 config2.scale_resolution_down_by *= 2;
8991
8992 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8993}
8994
8995TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8996 VideoStream config1 = DefaultConfig();
8997 VideoStream config2 = config1;
8998 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8999
9000 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9001}
9002
9003TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9004 VideoStream config1 = DefaultConfig();
9005 VideoStream config2 = config1;
9006 config2.scalability_mode = "L1T2";
9007
9008 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9009}
9010
Tommi62b01db2022-01-25 23:41:22 +01009011// Simple test that just creates and then immediately destroys an encoder.
9012// The purpose of the test is to make sure that nothing bad happens if the
9013// initialization step on the encoder queue, doesn't run.
9014TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9015 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9016 public:
9017 SuperLazyTaskQueue() = default;
9018 ~SuperLazyTaskQueue() override = default;
9019
9020 private:
9021 void Delete() override { delete this; }
9022 void PostTask(std::unique_ptr<QueuedTask> task) override {
9023 // meh.
9024 }
9025 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
9026 uint32_t milliseconds) override {
9027 ASSERT_TRUE(false);
9028 }
9029 };
9030
9031 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009032 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 23:41:22 +01009033 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
9034 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9035 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009036 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009037 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9038 time_controller.GetClock());
9039 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9040 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9041 CreateBuiltinVideoBitrateAllocatorFactory();
9042 VideoStreamEncoderSettings encoder_settings{
9043 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9044 encoder_settings.encoder_factory = &encoder_factory;
9045 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9046
9047 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9048 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9049
9050 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9051 encoder_queue(new SuperLazyTaskQueue());
9052
9053 // Construct a VideoStreamEncoder instance and let it go out of scope without
9054 // doing anything else (including calling Stop()). This should be fine since
9055 // the posted init task will simply be deleted.
9056 auto encoder = std::make_unique<VideoStreamEncoder>(
9057 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
9058 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
9059 std::move(adapter), std::move(encoder_queue),
9060 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009061 kVideoBitrateAllocation,
9062 field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009063}
9064
Markus Handellb4e96d42021-11-05 12:00:55 +01009065TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9066 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9067 auto* adapter_ptr = adapter.get();
9068 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009069 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9070 nullptr;
9071 EXPECT_CALL(*adapter_ptr, Initialize)
9072 .WillOnce(Invoke([&video_stream_encoder_callback](
9073 FrameCadenceAdapterInterface::Callback* callback) {
9074 video_stream_encoder_callback = callback;
9075 }));
9076 TaskQueueBase* encoder_queue = nullptr;
9077 auto video_stream_encoder =
9078 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009079
Markus Handelle59fee82021-12-23 09:29:23 +01009080 // First a call before we know the frame size and hence cannot compute the
9081 // number of simulcast layers.
9082 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9083 &FrameCadenceAdapterInterface::
9084 ZeroHertzModeParams::num_simulcast_layers,
9085 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009086 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009087 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009088 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9089 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009090 factory.DepleteTaskQueues();
9091
9092 // Then a call as we've computed the number of simulcast layers after a passed
9093 // frame.
9094 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9095 &FrameCadenceAdapterInterface::
9096 ZeroHertzModeParams::num_simulcast_layers,
9097 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009098 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009099 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009100 Mock::VerifyAndClearExpectations(adapter_ptr);
9101
Markus Handelle59fee82021-12-23 09:29:23 +01009102 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009103 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009104 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009105 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009106 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9107 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009108 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009109 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009110}
9111
9112TEST(VideoStreamEncoderFrameCadenceTest,
9113 ForwardsFramesIntoFrameCadenceAdapter) {
9114 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9115 auto* adapter_ptr = adapter.get();
9116 test::FrameForwarder video_source;
9117 SimpleVideoStreamEncoderFactory factory;
9118 auto video_stream_encoder = factory.Create(std::move(adapter));
9119 video_stream_encoder->SetSource(
9120 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9121
9122 EXPECT_CALL(*adapter_ptr, OnFrame);
9123 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9124 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009125 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009126}
9127
Markus Handellee225432021-11-29 12:35:12 +01009128TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9129 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9130 auto* adapter_ptr = adapter.get();
9131 test::FrameForwarder video_source;
9132 SimpleVideoStreamEncoderFactory factory;
9133 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9134 nullptr;
9135 EXPECT_CALL(*adapter_ptr, Initialize)
9136 .WillOnce(Invoke([&video_stream_encoder_callback](
9137 FrameCadenceAdapterInterface::Callback* callback) {
9138 video_stream_encoder_callback = callback;
9139 }));
9140 TaskQueueBase* encoder_queue = nullptr;
9141 auto video_stream_encoder =
9142 factory.Create(std::move(adapter), &encoder_queue);
9143
9144 // This is just to make the VSE operational. We'll feed a frame directly by
9145 // the callback interface.
9146 video_stream_encoder->SetSource(
9147 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9148
9149 VideoEncoderConfig video_encoder_config;
9150 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9151 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9152 /*max_data_payload_length=*/1000);
9153
9154 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9155 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009156 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009157 factory.DepleteTaskQueues();
9158}
9159
Markus Handell8d87c462021-12-16 11:37:16 +01009160TEST(VideoStreamEncoderFrameCadenceTest,
9161 DeactivatesActivatesLayersOnBitrateChanges) {
9162 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9163 auto* adapter_ptr = adapter.get();
9164 SimpleVideoStreamEncoderFactory factory;
9165 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9166 nullptr;
9167 EXPECT_CALL(*adapter_ptr, Initialize)
9168 .WillOnce(Invoke([&video_stream_encoder_callback](
9169 FrameCadenceAdapterInterface::Callback* callback) {
9170 video_stream_encoder_callback = callback;
9171 }));
9172 TaskQueueBase* encoder_queue = nullptr;
9173 auto video_stream_encoder =
9174 factory.Create(std::move(adapter), &encoder_queue);
9175
9176 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9177 // {150000, 450000}.
9178 VideoEncoderConfig video_encoder_config;
9179 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9180 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9181 kMaxPayloadLength);
9182 // Ensure an encoder is created.
9183 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9184
9185 // Both layers enabled at 1 MBit/s.
9186 video_stream_encoder->OnBitrateUpdated(
9187 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9188 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9189 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9190 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9191 factory.DepleteTaskQueues();
9192 Mock::VerifyAndClearExpectations(adapter_ptr);
9193
9194 // Layer 1 disabled at 200 KBit/s.
9195 video_stream_encoder->OnBitrateUpdated(
9196 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9197 DataRate::KilobitsPerSec(200), 0, 0, 0);
9198 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9199 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9200 factory.DepleteTaskQueues();
9201 Mock::VerifyAndClearExpectations(adapter_ptr);
9202
9203 // All layers off at suspended video.
9204 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9205 DataRate::Zero(), 0, 0, 0);
9206 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9207 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9208 factory.DepleteTaskQueues();
9209 Mock::VerifyAndClearExpectations(adapter_ptr);
9210
9211 // Both layers enabled again back at 1 MBit/s.
9212 video_stream_encoder->OnBitrateUpdated(
9213 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9214 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9215 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9216 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9217 factory.DepleteTaskQueues();
9218}
9219
9220TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9221 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9222 auto* adapter_ptr = adapter.get();
9223 SimpleVideoStreamEncoderFactory factory;
9224 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9225 nullptr;
9226 EXPECT_CALL(*adapter_ptr, Initialize)
9227 .WillOnce(Invoke([&video_stream_encoder_callback](
9228 FrameCadenceAdapterInterface::Callback* callback) {
9229 video_stream_encoder_callback = callback;
9230 }));
9231 TaskQueueBase* encoder_queue = nullptr;
9232 auto video_stream_encoder =
9233 factory.Create(std::move(adapter), &encoder_queue);
9234
9235 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9236 VideoEncoderConfig video_encoder_config;
9237 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9238 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9239 kMaxPayloadLength);
9240 video_stream_encoder->OnBitrateUpdated(
9241 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9242 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9243
9244 // Pass a frame which has unconverged results.
9245 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9246 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9247 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9248 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9249 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9250 CodecSpecificInfo codec_specific;
9251 codec_specific.codecType = kVideoCodecGeneric;
9252 return codec_specific;
9253 }));
9254 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9255 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9256 factory.DepleteTaskQueues();
9257 Mock::VerifyAndClearExpectations(adapter_ptr);
9258 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9259
9260 // Pass a frame which converges in layer 0 and not in layer 1.
9261 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9262 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9263 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9264 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9265 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9266 CodecSpecificInfo codec_specific;
9267 codec_specific.codecType = kVideoCodecGeneric;
9268 return codec_specific;
9269 }));
9270 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9271 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9272 factory.DepleteTaskQueues();
9273 Mock::VerifyAndClearExpectations(adapter_ptr);
9274 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9275}
9276
Markus Handell2e0f4f02021-12-21 19:14:58 +01009277TEST(VideoStreamEncoderFrameCadenceTest,
9278 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9279 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9280 auto* adapter_ptr = adapter.get();
9281 MockVideoSourceInterface mock_source;
9282 SimpleVideoStreamEncoderFactory factory;
9283 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9284 nullptr;
9285 EXPECT_CALL(*adapter_ptr, Initialize)
9286 .WillOnce(Invoke([&video_stream_encoder_callback](
9287 FrameCadenceAdapterInterface::Callback* callback) {
9288 video_stream_encoder_callback = callback;
9289 }));
9290 TaskQueueBase* encoder_queue = nullptr;
9291 auto video_stream_encoder =
9292 factory.Create(std::move(adapter), &encoder_queue);
9293 video_stream_encoder->SetSource(
9294 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9295 VideoEncoderConfig config;
9296 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9297 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9298 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9299 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9300 // Ensure the encoder is set up.
9301 factory.DepleteTaskQueues();
9302
Markus Handell818e7fb2021-12-30 13:01:33 +01009303 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9304 .WillOnce(Invoke([video_stream_encoder_callback] {
9305 video_stream_encoder_callback->RequestRefreshFrame();
9306 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009307 EXPECT_CALL(mock_source, RequestRefreshFrame);
9308 video_stream_encoder->SendKeyFrame();
9309 factory.DepleteTaskQueues();
9310 Mock::VerifyAndClearExpectations(adapter_ptr);
9311 Mock::VerifyAndClearExpectations(&mock_source);
9312
Markus Handell818e7fb2021-12-30 13:01:33 +01009313 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009314 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9315 video_stream_encoder->SendKeyFrame();
9316 factory.DepleteTaskQueues();
9317}
9318
Markus Handell818e7fb2021-12-30 13:01:33 +01009319TEST(VideoStreamEncoderFrameCadenceTest,
9320 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9321 SimpleVideoStreamEncoderFactory factory;
9322 auto encoder_queue =
9323 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9324 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9325
9326 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009327 test::ScopedKeyValueConfig field_trials(
9328 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009329 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009330 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9331 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009332 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9333
9334 MockVideoSourceInterface mock_source;
9335 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009336 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009337
9338 video_stream_encoder->SetSource(
9339 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9340 VideoEncoderConfig config;
9341 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9342 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9343 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9344
9345 // Eventually expect a refresh frame request when requesting a key frame
9346 // before initializing zero-hertz mode. This can happen in reality because the
9347 // threads invoking key frame requests and constraints setup aren't
9348 // synchronized.
9349 EXPECT_CALL(mock_source, RequestRefreshFrame);
9350 video_stream_encoder->SendKeyFrame();
9351 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9352 factory.DepleteTaskQueues();
9353}
9354
perkj26091b12016-09-01 01:17:40 -07009355} // namespace webrtc