blob: c1d89d73b29425b16a69c646f5a02b58cd9e2d47 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010022#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020023#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010024#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010025#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010026#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080027#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020029#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020030#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010031#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010032#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020033#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020034#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010035#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020036#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010037#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020038#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070039#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080040#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020041#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010042#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
43#include "modules/video_coding/codecs/h264/include/h264.h"
44#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
45#include "modules/video_coding/codecs/vp8/include/vp8.h"
46#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020047#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020048#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020049#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010050#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020051#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010052#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020053#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080055#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020056#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010057#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020058#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/encoder_settings.h"
60#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020061#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010065#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020066#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020067#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010068#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020069#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070070
71namespace webrtc {
72
sprang57c2fff2017-01-16 06:24:02 -080073using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020074using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020075using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Ge;
78using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010079using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020080using ::testing::Le;
81using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010082using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010083using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010085using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010087using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020088using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080089
perkj803d97f2016-11-01 11:45:46 -070090namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020091const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010092const int kQpLow = 1;
93const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020094const int kMinFramerateFps = 2;
95const int kMinBalancedFramerateFps = 7;
96const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080097const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020098const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
99const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
100const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
101const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800102const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700103const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200104const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200105const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800110
Asa Persson606d3cb2021-10-04 10:07:11 +0200111uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200112 0x00, 0x00, 0x03, 0x03, 0xF4,
113 0x05, 0x03, 0xC7, 0xE0, 0x1B,
114 0x41, 0x10, 0x8D, 0x00};
115
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100116const uint8_t kCodedFrameVp8Qp25[] = {
117 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
118 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
119 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
120
Markus Handell8d87c462021-12-16 11:37:16 +0100121void PassAFrame(
122 TaskQueueBase* encoder_queue,
123 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
124 int64_t ntp_time_ms) {
125 encoder_queue->PostTask(
126 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
127 video_stream_encoder_callback->OnFrame(
128 Timestamp::Millis(ntp_time_ms), 1,
129 VideoFrame::Builder()
130 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
131 /*width=*/16, /*height=*/16))
132 .build());
133 }));
134}
135
perkj803d97f2016-11-01 11:45:46 -0700136class TestBuffer : public webrtc::I420Buffer {
137 public:
138 TestBuffer(rtc::Event* event, int width, int height)
139 : I420Buffer(width, height), event_(event) {}
140
141 private:
142 friend class rtc::RefCountedObject<TestBuffer>;
143 ~TestBuffer() override {
144 if (event_)
145 event_->Set();
146 }
147 rtc::Event* const event_;
148};
149
Henrik Boström56db9ff2021-03-24 09:06:45 +0100150// A fake native buffer that can't be converted to I420. Upon scaling, it
151// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700152class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
153 public:
154 FakeNativeBuffer(rtc::Event* event, int width, int height)
155 : event_(event), width_(width), height_(height) {}
156 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
157 int width() const override { return width_; }
158 int height() const override { return height_; }
159 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
160 return nullptr;
161 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100162 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
163 int offset_x,
164 int offset_y,
165 int crop_width,
166 int crop_height,
167 int scaled_width,
168 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200169 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
170 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100171 }
Noah Richards51db4212019-06-12 06:59:12 -0700172
173 private:
174 friend class rtc::RefCountedObject<FakeNativeBuffer>;
175 ~FakeNativeBuffer() override {
176 if (event_)
177 event_->Set();
178 }
179 rtc::Event* const event_;
180 const int width_;
181 const int height_;
182};
183
Evan Shrubsole895556e2020-10-05 09:15:13 +0200184// A fake native buffer that is backed by an NV12 buffer.
185class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
186 public:
187 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
188 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
189
190 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
191 int width() const override { return nv12_buffer_->width(); }
192 int height() const override { return nv12_buffer_->height(); }
193 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
194 return nv12_buffer_->ToI420();
195 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200196 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
197 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
198 if (absl::c_find(types, Type::kNV12) != types.end()) {
199 return nv12_buffer_;
200 }
201 return nullptr;
202 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200203 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
204
205 private:
206 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
207 ~FakeNV12NativeBuffer() override {
208 if (event_)
209 event_->Set();
210 }
211 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
212 rtc::Event* const event_;
213};
214
Niels Möller7dc26b72017-12-06 10:27:48 +0100215class CpuOveruseDetectorProxy : public OveruseFrameDetector {
216 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200217 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
218 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200219 last_target_framerate_fps_(-1),
220 framerate_updated_event_(true /* manual_reset */,
221 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100222 virtual ~CpuOveruseDetectorProxy() {}
223
224 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200225 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100226 last_target_framerate_fps_ = framerate_fps;
227 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200228 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 }
230
231 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200232 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100233 return last_target_framerate_fps_;
234 }
235
Niels Möller4db138e2018-04-19 09:04:13 +0200236 CpuOveruseOptions GetOptions() { return options_; }
237
Henrik Boström381d1092020-05-12 18:49:07 +0200238 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
239
Niels Möller7dc26b72017-12-06 10:27:48 +0100240 private:
Markus Handella3765182020-07-08 13:13:32 +0200241 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100242 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200243 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100244};
245
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200246class FakeVideoSourceRestrictionsListener
247 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200248 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200249 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200250 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200251 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200252 RTC_DCHECK(was_restrictions_updated_);
253 }
254
255 rtc::Event* restrictions_updated_event() {
256 return &restrictions_updated_event_;
257 }
258
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200259 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200260 void OnVideoSourceRestrictionsUpdated(
261 VideoSourceRestrictions restrictions,
262 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200263 rtc::scoped_refptr<Resource> reason,
264 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200265 was_restrictions_updated_ = true;
266 restrictions_updated_event_.Set();
267 }
268
269 private:
270 bool was_restrictions_updated_;
271 rtc::Event restrictions_updated_event_;
272};
273
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200274auto WantsFps(Matcher<int> fps_matcher) {
275 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
276 fps_matcher);
277}
278
279auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
280 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
281 AllOf(max_pixel_matcher, Gt(0)));
282}
283
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284auto ResolutionMax() {
285 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200286 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200287 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
288 Eq(absl::nullopt)));
289}
290
291auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200292 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200293}
294
295auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200296 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200297}
298
299auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200300 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200301}
302
303auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200304 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200305}
306
307auto FpsMaxResolutionMax() {
308 return AllOf(FpsMax(), ResolutionMax());
309}
310
311auto UnlimitedSinkWants() {
312 return AllOf(FpsUnlimited(), ResolutionMax());
313}
314
315auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
316 Matcher<int> fps_range_matcher;
317
318 if (last_frame_pixels <= 320 * 240) {
319 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200320 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200321 fps_range_matcher = AllOf(Ge(10), Le(15));
322 } else if (last_frame_pixels <= 640 * 480) {
323 fps_range_matcher = Ge(15);
324 } else {
325 fps_range_matcher = Eq(kDefaultFramerate);
326 }
327 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
328 fps_range_matcher);
329}
330
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200331auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
332 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
333 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
334}
335
336auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
337 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
338}
339
340auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
342}
343
344auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
346 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
347}
348
349auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
351 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
352}
353
354auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
355 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
356 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
357}
358
359auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
360 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
361 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
362}
363
mflodmancc3d4422017-08-03 08:27:51 -0700364class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700365 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100366 VideoStreamEncoderUnderTest(
367 TimeController* time_controller,
368 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
369 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
370 encoder_queue,
371 SendStatisticsProxy* stats_proxy,
372 const VideoStreamEncoderSettings& settings,
373 VideoStreamEncoder::BitrateAllocationCallbackType
374 allocation_callback_type)
375 : VideoStreamEncoder(time_controller->GetClock(),
376 1 /* number_of_cores */,
377 stats_proxy,
378 settings,
379 std::unique_ptr<OveruseFrameDetector>(
380 overuse_detector_proxy_ =
381 new CpuOveruseDetectorProxy(stats_proxy)),
382 std::move(cadence_adapter),
383 std::move(encoder_queue),
384 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200385 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200386 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200387 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200388 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200389 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200390 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200391 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200392 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100393 }
perkj803d97f2016-11-01 11:45:46 -0700394
Henrik Boström381d1092020-05-12 18:49:07 +0200395 void SetSourceAndWaitForRestrictionsUpdated(
396 rtc::VideoSourceInterface<VideoFrame>* source,
397 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200398 FakeVideoSourceRestrictionsListener listener;
399 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200400 SetSource(source, degradation_preference);
401 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200402 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200403 }
404
405 void SetSourceAndWaitForFramerateUpdated(
406 rtc::VideoSourceInterface<VideoFrame>* source,
407 const DegradationPreference& degradation_preference) {
408 overuse_detector_proxy_->framerate_updated_event()->Reset();
409 SetSource(source, degradation_preference);
410 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
411 }
412
413 void OnBitrateUpdatedAndWaitForManagedResources(
414 DataRate target_bitrate,
415 DataRate stable_target_bitrate,
416 DataRate link_allocation,
417 uint8_t fraction_lost,
418 int64_t round_trip_time_ms,
419 double cwnd_reduce_ratio) {
420 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
421 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
422 // Bitrate is updated on the encoder queue.
423 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200424 }
425
kthelgason2fc52542017-03-03 00:24:41 -0800426 // This is used as a synchronisation mechanism, to make sure that the
427 // encoder queue is not blocked before we start sending it frames.
428 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100429 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800430 }
431
Henrik Boström91aa7322020-04-28 12:24:33 +0200432 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200433 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200434 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200435 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200436 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200437 event.Set();
438 });
439 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100440 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200442
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 void TriggerCpuUnderuse() {
444 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200445 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200446 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200447 event.Set();
448 });
449 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100450 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200451 }
kthelgason876222f2016-11-29 01:44:11 -0800452
Henrik Boström91aa7322020-04-28 12:24:33 +0200453 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200454 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200456 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200457 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200458 event.Set();
459 });
460 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100461 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200462 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200463 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200464 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200465 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200466 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200467 event.Set();
468 });
469 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100470 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200471 }
472
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200473 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100474 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200475 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
476 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200477 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700478};
479
Noah Richards51db4212019-06-12 06:59:12 -0700480// Simulates simulcast behavior and makes highest stream resolutions divisible
481// by 4.
482class CroppingVideoStreamFactory
483 : public VideoEncoderConfig::VideoStreamFactoryInterface {
484 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200485 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700486
487 private:
488 std::vector<VideoStream> CreateEncoderStreams(
489 int width,
490 int height,
491 const VideoEncoderConfig& encoder_config) override {
492 std::vector<VideoStream> streams = test::CreateVideoStreams(
493 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700494 return streams;
495 }
Noah Richards51db4212019-06-12 06:59:12 -0700496};
497
sprangb1ca0732017-02-01 08:38:12 -0800498class AdaptingFrameForwarder : public test::FrameForwarder {
499 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200500 explicit AdaptingFrameForwarder(TimeController* time_controller)
501 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700502 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800503
504 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200505 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800506 adaptation_enabled_ = enabled;
507 }
508
asaperssonfab67072017-04-04 05:51:49 -0700509 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200510 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800511 return adaptation_enabled_;
512 }
513
Henrik Boström1124ed12021-02-25 10:30:39 +0100514 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
515 // the resolution or frame rate was different than it is currently. If
516 // something else is modified, such as encoder resolutions, but the resolution
517 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700518 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200519 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700520 return last_wants_;
521 }
522
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200523 absl::optional<int> last_sent_width() const { return last_width_; }
524 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800525
sprangb1ca0732017-02-01 08:38:12 -0800526 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200527 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100528 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200529
sprangb1ca0732017-02-01 08:38:12 -0800530 int cropped_width = 0;
531 int cropped_height = 0;
532 int out_width = 0;
533 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700534 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000535 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
536 << "w=" << video_frame.width()
537 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700538 if (adapter_.AdaptFrameResolution(
539 video_frame.width(), video_frame.height(),
540 video_frame.timestamp_us() * 1000, &cropped_width,
541 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100542 VideoFrame adapted_frame =
543 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200544 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100545 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200546 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100547 .set_timestamp_ms(99)
548 .set_rotation(kVideoRotation_0)
549 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100550 if (video_frame.has_update_rect()) {
551 adapted_frame.set_update_rect(
552 video_frame.update_rect().ScaleWithFrame(
553 video_frame.width(), video_frame.height(), 0, 0,
554 video_frame.width(), video_frame.height(), out_width,
555 out_height));
556 }
sprangc5d62e22017-04-02 23:53:04 -0700557 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800558 last_width_.emplace(adapted_frame.width());
559 last_height_.emplace(adapted_frame.height());
560 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200561 last_width_ = absl::nullopt;
562 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700563 }
sprangb1ca0732017-02-01 08:38:12 -0800564 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000565 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800566 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800567 last_width_.emplace(video_frame.width());
568 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800569 }
570 }
571
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200572 void OnOutputFormatRequest(int width, int height) {
573 absl::optional<std::pair<int, int>> target_aspect_ratio =
574 std::make_pair(width, height);
575 absl::optional<int> max_pixel_count = width * height;
576 absl::optional<int> max_fps;
577 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
578 max_fps);
579 }
580
sprangb1ca0732017-02-01 08:38:12 -0800581 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
582 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200583 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100584 rtc::VideoSinkWants prev_wants = sink_wants_locked();
585 bool did_adapt =
586 prev_wants.max_pixel_count != wants.max_pixel_count ||
587 prev_wants.target_pixel_count != wants.target_pixel_count ||
588 prev_wants.max_framerate_fps != wants.max_framerate_fps;
589 if (did_adapt) {
590 last_wants_ = prev_wants;
591 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100592 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200593 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800594 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200595
596 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800597 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200598 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
599 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200600 absl::optional<int> last_width_;
601 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800602};
sprangc5d62e22017-04-02 23:53:04 -0700603
Niels Möller213618e2018-07-24 09:29:58 +0200604// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700605class MockableSendStatisticsProxy : public SendStatisticsProxy {
606 public:
607 MockableSendStatisticsProxy(Clock* clock,
608 const VideoSendStream::Config& config,
609 VideoEncoderConfig::ContentType content_type)
610 : SendStatisticsProxy(clock, config, content_type) {}
611
612 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200613 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700614 if (mock_stats_)
615 return *mock_stats_;
616 return SendStatisticsProxy::GetStats();
617 }
618
Niels Möller213618e2018-07-24 09:29:58 +0200619 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200620 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200621 if (mock_stats_)
622 return mock_stats_->input_frame_rate;
623 return SendStatisticsProxy::GetInputFrameRate();
624 }
sprangc5d62e22017-04-02 23:53:04 -0700625 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200626 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700627 mock_stats_.emplace(stats);
628 }
629
630 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200631 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700632 mock_stats_.reset();
633 }
634
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200635 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
636 on_frame_dropped_ = std::move(callback);
637 }
638
sprangc5d62e22017-04-02 23:53:04 -0700639 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200640 void OnFrameDropped(DropReason reason) override {
641 SendStatisticsProxy::OnFrameDropped(reason);
642 if (on_frame_dropped_)
643 on_frame_dropped_(reason);
644 }
645
Markus Handella3765182020-07-08 13:13:32 +0200646 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200647 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200648 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700649};
650
Markus Handellb4e96d42021-11-05 12:00:55 +0100651class SimpleVideoStreamEncoderFactory {
652 public:
653 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
654 public:
655 using VideoStreamEncoder::VideoStreamEncoder;
656 ~AdaptedVideoStreamEncoder() { Stop(); }
657 };
658
Markus Handell8d87c462021-12-16 11:37:16 +0100659 class MockFakeEncoder : public test::FakeEncoder {
660 public:
661 using FakeEncoder::FakeEncoder;
662 MOCK_METHOD(CodecSpecificInfo,
663 EncodeHook,
664 (EncodedImage & encoded_image,
665 rtc::scoped_refptr<EncodedImageBuffer> buffer),
666 (override));
667 };
668
Markus Handellee225432021-11-29 12:35:12 +0100669 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100670 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100671 encoder_settings_.bitrate_allocator_factory =
672 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100673 }
674
675 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
Markus Handellee225432021-11-29 12:35:12 +0100676 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
677 TaskQueueBase** encoder_queue_ptr = nullptr) {
678 auto encoder_queue =
679 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
680 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
681 if (encoder_queue_ptr)
682 *encoder_queue_ptr = encoder_queue.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100683 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
684 time_controller_.GetClock(),
685 /*number_of_cores=*/1,
686 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
687 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100688 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100689 VideoStreamEncoder::BitrateAllocationCallbackType::
690 kVideoBitrateAllocation);
691 result->SetSink(&sink_, /*rotation_applied=*/false);
692 return result;
693 }
694
Markus Handell9a478b52021-11-18 16:07:01 +0100695 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100696 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100697
Markus Handellb4e96d42021-11-05 12:00:55 +0100698 private:
699 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
700 public:
701 ~NullEncoderSink() override = default;
702 void OnEncoderConfigurationChanged(
703 std::vector<VideoStream> streams,
704 bool is_svc,
705 VideoEncoderConfig::ContentType content_type,
706 int min_transmit_bitrate_bps) override {}
707 void OnBitrateAllocationUpdated(
708 const VideoBitrateAllocation& allocation) override {}
709 void OnVideoLayersAllocationUpdated(
710 VideoLayersAllocation allocation) override {}
711 Result OnEncodedImage(
712 const EncodedImage& encoded_image,
713 const CodecSpecificInfo* codec_specific_info) override {
714 return Result(EncodedImageCallback::Result::OK);
715 }
716 };
717
Markus Handellee225432021-11-29 12:35:12 +0100718 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
719 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
720 time_controller_.CreateTaskQueueFactory()};
721 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
722 std::make_unique<MockableSendStatisticsProxy>(
723 time_controller_.GetClock(),
724 VideoSendStream::Config(nullptr),
725 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
726 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
727 CreateBuiltinVideoBitrateAllocatorFactory();
728 VideoStreamEncoderSettings encoder_settings_{
729 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100730 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
731 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100732 NullEncoderSink sink_;
733};
734
735class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
736 public:
737 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100738 MOCK_METHOD(void,
739 SetZeroHertzModeEnabled,
740 (absl::optional<ZeroHertzModeParams>),
741 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100742 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100743 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
744 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100745 MOCK_METHOD(void,
746 UpdateLayerQualityConvergence,
747 (int spatial_index, bool converged),
748 (override));
749 MOCK_METHOD(void,
750 UpdateLayerStatus,
751 (int spatial_index, bool enabled),
752 (override));
Markus Handell2e0f4f02021-12-21 19:14:58 +0100753 MOCK_METHOD(bool, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100754};
755
philipel9b058032020-02-10 11:30:00 +0100756class MockEncoderSelector
757 : public VideoEncoderFactory::EncoderSelectorInterface {
758 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200759 MOCK_METHOD(void,
760 OnCurrentEncoder,
761 (const SdpVideoFormat& format),
762 (override));
763 MOCK_METHOD(absl::optional<SdpVideoFormat>,
764 OnAvailableBitrate,
765 (const DataRate& rate),
766 (override));
767 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100768};
769
Markus Handell2e0f4f02021-12-21 19:14:58 +0100770class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
771 public:
772 MOCK_METHOD(void,
773 AddOrUpdateSink,
774 (rtc::VideoSinkInterface<VideoFrame>*,
775 const rtc::VideoSinkWants&),
776 (override));
777 MOCK_METHOD(void,
778 RemoveSink,
779 (rtc::VideoSinkInterface<VideoFrame>*),
780 (override));
781 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
782};
783
perkj803d97f2016-11-01 11:45:46 -0700784} // namespace
785
mflodmancc3d4422017-08-03 08:27:51 -0700786class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700787 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200788 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700789
mflodmancc3d4422017-08-03 08:27:51 -0700790 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700791 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700792 codec_width_(320),
793 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200794 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200795 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200796 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700797 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200798 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700799 video_send_config_,
800 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200801 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700802
803 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700804 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700805 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200806 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800807 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200808 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200809 video_send_config_.rtp.payload_name = "FAKE";
810 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700811
Per512ecb32016-09-23 15:52:06 +0200812 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200813 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200814 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
815 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
816 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100817 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700818
Niels Möllerf1338562018-04-26 09:51:47 +0200819 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800820 }
821
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100822 void ConfigureEncoder(
823 VideoEncoderConfig video_encoder_config,
824 VideoStreamEncoder::BitrateAllocationCallbackType
825 allocation_callback_type =
826 VideoStreamEncoder::BitrateAllocationCallbackType::
827 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700828 if (video_stream_encoder_)
829 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100830
831 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
832 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
833 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
834 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
835 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
836 encoder_queue_ptr);
837 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
838 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
839 stats_proxy_.get(), video_send_config_.encoder_settings,
840 allocation_callback_type);
Asa Persson606d3cb2021-10-04 10:07:11 +0200841 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700843 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200844 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700845 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200846 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800848 }
849
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100850 void ResetEncoder(const std::string& payload_name,
851 size_t num_streams,
852 size_t num_temporal_layers,
853 unsigned char num_spatial_layers,
854 bool screenshare,
855 VideoStreamEncoder::BitrateAllocationCallbackType
856 allocation_callback_type =
857 VideoStreamEncoder::BitrateAllocationCallbackType::
858 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200859 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800860
861 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200862 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
863 num_streams, &video_encoder_config);
864 for (auto& layer : video_encoder_config.simulcast_layers) {
865 layer.num_temporal_layers = num_temporal_layers;
866 layer.max_framerate = kDefaultFramerate;
867 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100868 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200869 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700870 video_encoder_config.content_type =
871 screenshare ? VideoEncoderConfig::ContentType::kScreen
872 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700873 if (payload_name == "VP9") {
874 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
875 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200876 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700877 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200878 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
879 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700880 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100881 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700882 }
883
sprang57c2fff2017-01-16 06:24:02 -0800884 VideoFrame CreateFrame(int64_t ntp_time_ms,
885 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200886 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200887 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200888 destruction_event, codec_width_, codec_height_))
889 .set_ntp_time_ms(ntp_time_ms)
890 .set_timestamp_ms(99)
891 .set_rotation(kVideoRotation_0)
892 .build();
perkj26091b12016-09-01 01:17:40 -0700893 }
894
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100895 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
896 rtc::Event* destruction_event,
897 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200898 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200899 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200900 destruction_event, codec_width_, codec_height_))
901 .set_ntp_time_ms(ntp_time_ms)
902 .set_timestamp_ms(99)
903 .set_rotation(kVideoRotation_0)
904 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
905 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100906 }
907
sprang57c2fff2017-01-16 06:24:02 -0800908 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200909 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
910 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200911 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200912 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200913 .set_ntp_time_ms(ntp_time_ms)
914 .set_timestamp_ms(ntp_time_ms)
915 .set_rotation(kVideoRotation_0)
916 .build();
perkj803d97f2016-11-01 11:45:46 -0700917 }
918
Evan Shrubsole895556e2020-10-05 09:15:13 +0200919 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200920 return VideoFrame::Builder()
921 .set_video_frame_buffer(NV12Buffer::Create(width, height))
922 .set_ntp_time_ms(ntp_time_ms)
923 .set_timestamp_ms(ntp_time_ms)
924 .set_rotation(kVideoRotation_0)
925 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200926 }
927
Noah Richards51db4212019-06-12 06:59:12 -0700928 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
929 rtc::Event* destruction_event,
930 int width,
931 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200932 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200933 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200934 destruction_event, width, height))
935 .set_ntp_time_ms(ntp_time_ms)
936 .set_timestamp_ms(99)
937 .set_rotation(kVideoRotation_0)
938 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700939 }
940
Evan Shrubsole895556e2020-10-05 09:15:13 +0200941 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
942 rtc::Event* destruction_event,
943 int width,
944 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200945 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200946 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200947 destruction_event, width, height))
948 .set_ntp_time_ms(ntp_time_ms)
949 .set_timestamp_ms(99)
950 .set_rotation(kVideoRotation_0)
951 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200952 }
953
Noah Richards51db4212019-06-12 06:59:12 -0700954 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
955 rtc::Event* destruction_event) const {
956 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
957 codec_height_);
958 }
959
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100960 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200962 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100963
964 video_source_.IncomingCapturedFrame(
965 CreateFrame(1, codec_width_, codec_height_));
966 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200967 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100968 }
969
sprang4847ae62017-06-27 07:06:52 -0700970 void WaitForEncodedFrame(int64_t expected_ntp_time) {
971 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200972 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700973 }
974
975 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
976 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200977 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700978 return ok;
979 }
980
981 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
982 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200983 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700984 }
985
986 void ExpectDroppedFrame() {
987 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200988 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700989 }
990
991 bool WaitForFrame(int64_t timeout_ms) {
992 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200993 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700994 return ok;
995 }
996
perkj26091b12016-09-01 01:17:40 -0700997 class TestEncoder : public test::FakeEncoder {
998 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200999 explicit TestEncoder(TimeController* time_controller)
1000 : FakeEncoder(time_controller->GetClock()),
1001 time_controller_(time_controller) {
1002 RTC_DCHECK(time_controller_);
1003 }
perkj26091b12016-09-01 01:17:40 -07001004
Erik Språngaed30702018-11-05 12:57:17 +01001005 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001006 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001007 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001008 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001009 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001010 info.scaling_settings = VideoEncoder::ScalingSettings(
1011 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001012 }
1013 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001014 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1015 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001016 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001017 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001018 for (int tid = 0; tid < num_layers; ++tid)
1019 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001020 }
1021 }
Erik Språngaed30702018-11-05 12:57:17 +01001022 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001023
1024 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001025 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001026 info.apply_alignment_to_all_simulcast_layers =
1027 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001028 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001029 if (is_qp_trusted_.has_value()) {
1030 info.is_qp_trusted = is_qp_trusted_;
1031 }
Erik Språngaed30702018-11-05 12:57:17 +01001032 return info;
kthelgason876222f2016-11-29 01:44:11 -08001033 }
1034
Erik Språngb7cb7b52019-02-26 15:52:33 +01001035 int32_t RegisterEncodeCompleteCallback(
1036 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001037 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001038 encoded_image_callback_ = callback;
1039 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1040 }
1041
perkjfa10b552016-10-02 23:45:26 -07001042 void ContinueEncode() { continue_encode_event_.Set(); }
1043
1044 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1045 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001046 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001047 EXPECT_EQ(timestamp_, timestamp);
1048 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1049 }
1050
kthelgason2fc52542017-03-03 00:24:41 -08001051 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001052 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001053 quality_scaling_ = b;
1054 }
kthelgasonad9010c2017-02-14 00:46:51 -08001055
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001056 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001057 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001058 requested_resolution_alignment_ = requested_resolution_alignment;
1059 }
1060
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001061 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1062 MutexLock lock(&local_mutex_);
1063 apply_alignment_to_all_simulcast_layers_ = b;
1064 }
1065
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001066 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001067 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001068 is_hardware_accelerated_ = is_hardware_accelerated;
1069 }
1070
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001071 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1072 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001073 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001074 temporal_layers_supported_[spatial_idx] = supported;
1075 }
1076
Sergey Silkin6456e352019-07-08 17:56:40 +02001077 void SetResolutionBitrateLimits(
1078 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001079 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001080 resolution_bitrate_limits_ = thresholds;
1081 }
1082
sprangfe627f32017-03-29 08:24:59 -07001083 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001085 force_init_encode_failed_ = force_failure;
1086 }
1087
Niels Möller6bb5ab92019-01-11 11:11:10 +01001088 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001089 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001090 rate_factor_ = rate_factor;
1091 }
1092
Erik Språngd7329ca2019-02-21 21:19:53 +01001093 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001094 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001095 return last_framerate_;
1096 }
1097
Erik Språngd7329ca2019-02-21 21:19:53 +01001098 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001099 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001100 return last_update_rect_;
1101 }
1102
Niels Möller87e2d782019-03-07 10:18:23 +01001103 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001104 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001105 return last_frame_types_;
1106 }
1107
1108 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001109 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001110 keyframe ? VideoFrameType::kVideoFrameKey
1111 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001112 {
Markus Handella3765182020-07-08 13:13:32 +02001113 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001114 last_frame_types_ = frame_type;
1115 }
Niels Möllerb859b322019-03-07 12:40:01 +01001116 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001117 }
1118
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001119 void InjectEncodedImage(const EncodedImage& image,
1120 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001121 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001122 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001123 }
1124
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001125 void SetEncodedImageData(
1126 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001127 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001128 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001129 }
1130
Erik Språngd7329ca2019-02-21 21:19:53 +01001131 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001132 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 expect_null_frame_ = true;
1134 }
1135
Erik Språng5056af02019-09-02 15:53:11 +02001136 absl::optional<VideoEncoder::RateControlParameters>
1137 GetAndResetLastRateControlSettings() {
1138 auto settings = last_rate_control_settings_;
1139 last_rate_control_settings_.reset();
1140 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001141 }
1142
Henrik Boström56db9ff2021-03-24 09:06:45 +01001143 int GetLastInputWidth() const {
1144 MutexLock lock(&local_mutex_);
1145 return last_input_width_;
1146 }
1147
1148 int GetLastInputHeight() const {
1149 MutexLock lock(&local_mutex_);
1150 return last_input_height_;
1151 }
1152
Evan Shrubsole895556e2020-10-05 09:15:13 +02001153 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1154 MutexLock lock(&local_mutex_);
1155 return last_input_pixel_format_;
1156 }
1157
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001158 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001159 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001160 return num_set_rates_;
1161 }
1162
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001163 void SetPreferredPixelFormats(
1164 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1165 pixel_formats) {
1166 MutexLock lock(&local_mutex_);
1167 preferred_pixel_formats_ = std::move(pixel_formats);
1168 }
1169
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001170 void SetIsQpTrusted(absl::optional<bool> trusted) {
1171 MutexLock lock(&local_mutex_);
1172 is_qp_trusted_ = trusted;
1173 }
1174
perkjfa10b552016-10-02 23:45:26 -07001175 private:
perkj26091b12016-09-01 01:17:40 -07001176 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001177 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001178 {
Markus Handella3765182020-07-08 13:13:32 +02001179 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001180 if (expect_null_frame_) {
1181 EXPECT_EQ(input_image.timestamp(), 0u);
1182 EXPECT_EQ(input_image.width(), 1);
1183 last_frame_types_ = *frame_types;
1184 expect_null_frame_ = false;
1185 } else {
1186 EXPECT_GT(input_image.timestamp(), timestamp_);
1187 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1188 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1189 }
perkj26091b12016-09-01 01:17:40 -07001190
1191 timestamp_ = input_image.timestamp();
1192 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001193 last_input_width_ = input_image.width();
1194 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001195 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001196 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001197 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001198 }
Niels Möllerb859b322019-03-07 12:40:01 +01001199 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001200 return result;
1201 }
1202
Niels Möller08ae7ce2020-09-23 15:58:12 +02001203 CodecSpecificInfo EncodeHook(
1204 EncodedImage& encoded_image,
1205 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001206 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001207 {
1208 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001209 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001210 }
1211 MutexLock lock(&local_mutex_);
1212 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001213 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001214 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001215 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001216 }
1217
sprangfe627f32017-03-29 08:24:59 -07001218 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001219 const Settings& settings) override {
1220 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001221
Markus Handella3765182020-07-08 13:13:32 +02001222 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001223 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001224
Erik Språng82fad3d2018-03-21 09:57:23 +01001225 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001226 // Simulate setting up temporal layers, in order to validate the life
1227 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001228 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001229 frame_buffer_controller_ =
1230 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001231 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001232 if (force_init_encode_failed_) {
1233 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001234 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001235 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001236
Erik Språngb7cb7b52019-02-26 15:52:33 +01001237 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001238 return res;
1239 }
1240
Erik Språngb7cb7b52019-02-26 15:52:33 +01001241 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001242 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001243 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1244 initialized_ = EncoderState::kUninitialized;
1245 return FakeEncoder::Release();
1246 }
1247
Erik Språng16cb8f52019-04-12 13:59:09 +02001248 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001249 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001250 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001251 VideoBitrateAllocation adjusted_rate_allocation;
1252 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1253 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001254 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001255 adjusted_rate_allocation.SetBitrate(
1256 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001257 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001258 rate_factor_));
1259 }
1260 }
1261 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001262 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001263 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001264 RateControlParameters adjusted_paramters = parameters;
1265 adjusted_paramters.bitrate = adjusted_rate_allocation;
1266 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001267 }
1268
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001269 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001270 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001271 enum class EncoderState {
1272 kUninitialized,
1273 kInitializationFailed,
1274 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001275 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001276 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001277 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1278 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1279 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1280 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1281 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1282 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001283 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1284 false;
Markus Handella3765182020-07-08 13:13:32 +02001285 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001286 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1287 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001288 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001289 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001290 absl::optional<bool>
1291 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001292 local_mutex_);
1293 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1294 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1295 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001296 absl::optional<VideoEncoder::RateControlParameters>
1297 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001298 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1299 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001300 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001301 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001302 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1303 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001304 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001305 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001306 RTC_GUARDED_BY(local_mutex_);
1307 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001308 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1309 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001310 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1311 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001312 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001313 };
1314
mflodmancc3d4422017-08-03 08:27:51 -07001315 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001316 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001317 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1318 : time_controller_(time_controller), test_encoder_(test_encoder) {
1319 RTC_DCHECK(time_controller_);
1320 }
perkj26091b12016-09-01 01:17:40 -07001321
perkj26091b12016-09-01 01:17:40 -07001322 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001323 EXPECT_TRUE(
1324 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1325 }
1326
1327 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1328 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001329 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001330 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001331 return false;
perkj26091b12016-09-01 01:17:40 -07001332 {
Markus Handella3765182020-07-08 13:13:32 +02001333 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001334 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001335 }
1336 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001337 return true;
perkj26091b12016-09-01 01:17:40 -07001338 }
1339
sprangb1ca0732017-02-01 08:38:12 -08001340 void WaitForEncodedFrame(uint32_t expected_width,
1341 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001342 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001343 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001344 }
1345
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001346 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001347 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001348 uint32_t width = 0;
1349 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001350 {
Markus Handella3765182020-07-08 13:13:32 +02001351 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001352 width = last_width_;
1353 height = last_height_;
1354 }
1355 EXPECT_EQ(expected_height, height);
1356 EXPECT_EQ(expected_width, width);
1357 }
1358
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001359 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1360 VideoRotation rotation;
1361 {
Markus Handella3765182020-07-08 13:13:32 +02001362 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001363 rotation = last_rotation_;
1364 }
1365 EXPECT_EQ(expected_rotation, rotation);
1366 }
1367
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001368 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001369
sprangc5d62e22017-04-02 23:53:04 -07001370 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001371 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001372 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001373 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001374 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001375 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001376 }
1377
perkj26091b12016-09-01 01:17:40 -07001378 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001379 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001380 expect_frames_ = false;
1381 }
1382
asaperssonfab67072017-04-04 05:51:49 -07001383 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001384 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001385 return number_of_reconfigurations_;
1386 }
1387
asaperssonfab67072017-04-04 05:51:49 -07001388 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001389 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001390 return min_transmit_bitrate_bps_;
1391 }
1392
Erik Språngd7329ca2019-02-21 21:19:53 +01001393 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001394 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001395 num_expected_layers_ = num_layers;
1396 }
1397
Erik Språngb7cb7b52019-02-26 15:52:33 +01001398 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001399 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001400 return last_capture_time_ms_;
1401 }
1402
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001403 const EncodedImage& GetLastEncodedImage() {
1404 MutexLock lock(&mutex_);
1405 return last_encoded_image_;
1406 }
1407
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001408 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001409 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001410 return std::move(last_encoded_image_data_);
1411 }
1412
Per Kjellanderdcef6412020-10-07 15:09:05 +02001413 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1414 MutexLock lock(&mutex_);
1415 return last_bitrate_allocation_;
1416 }
1417
1418 int number_of_bitrate_allocations() const {
1419 MutexLock lock(&mutex_);
1420 return number_of_bitrate_allocations_;
1421 }
1422
Per Kjellandera9434842020-10-15 17:53:22 +02001423 VideoLayersAllocation GetLastVideoLayersAllocation() {
1424 MutexLock lock(&mutex_);
1425 return last_layers_allocation_;
1426 }
1427
1428 int number_of_layers_allocations() const {
1429 MutexLock lock(&mutex_);
1430 return number_of_layers_allocations_;
1431 }
1432
perkj26091b12016-09-01 01:17:40 -07001433 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001434 Result OnEncodedImage(
1435 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001436 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001437 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001438 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001439 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001440 last_encoded_image_data_ = std::vector<uint8_t>(
1441 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001442 uint32_t timestamp = encoded_image.Timestamp();
1443 if (last_timestamp_ != timestamp) {
1444 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001445 last_width_ = encoded_image._encodedWidth;
1446 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001447 } else {
1448 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001449 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1450 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001451 }
1452 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001453 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001454 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001455 if (num_received_layers_ == num_expected_layers_) {
1456 encoded_frame_event_.Set();
1457 }
sprangb1ca0732017-02-01 08:38:12 -08001458 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001459 }
1460
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001461 void OnEncoderConfigurationChanged(
1462 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001463 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001464 VideoEncoderConfig::ContentType content_type,
1465 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001466 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001467 ++number_of_reconfigurations_;
1468 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1469 }
1470
Per Kjellanderdcef6412020-10-07 15:09:05 +02001471 void OnBitrateAllocationUpdated(
1472 const VideoBitrateAllocation& allocation) override {
1473 MutexLock lock(&mutex_);
1474 ++number_of_bitrate_allocations_;
1475 last_bitrate_allocation_ = allocation;
1476 }
1477
Per Kjellandera9434842020-10-15 17:53:22 +02001478 void OnVideoLayersAllocationUpdated(
1479 VideoLayersAllocation allocation) override {
1480 MutexLock lock(&mutex_);
1481 ++number_of_layers_allocations_;
1482 last_layers_allocation_ = allocation;
1483 rtc::StringBuilder log;
1484 for (const auto& layer : allocation.active_spatial_layers) {
1485 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1486 << "[";
1487 for (const auto target_bitrate :
1488 layer.target_bitrate_per_temporal_layer) {
1489 log << target_bitrate.kbps() << ",";
1490 }
1491 log << "]";
1492 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001493 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001494 }
1495
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001496 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001497 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001498 TestEncoder* test_encoder_;
1499 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001500 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001501 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001502 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001503 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001504 uint32_t last_height_ = 0;
1505 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001506 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001507 size_t num_expected_layers_ = 1;
1508 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001509 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001510 int number_of_reconfigurations_ = 0;
1511 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001512 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1513 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001514 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1515 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001516 };
1517
Sergey Silkin5ee69672019-07-02 14:18:34 +02001518 class VideoBitrateAllocatorProxyFactory
1519 : public VideoBitrateAllocatorFactory {
1520 public:
1521 VideoBitrateAllocatorProxyFactory()
1522 : bitrate_allocator_factory_(
1523 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1524
1525 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1526 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001527 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001528 codec_config_ = codec;
1529 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1530 }
1531
1532 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001533 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001534 return codec_config_;
1535 }
1536
1537 private:
1538 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1539
Markus Handella3765182020-07-08 13:13:32 +02001540 mutable Mutex mutex_;
1541 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001542 };
1543
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001544 Clock* clock() { return time_controller_.GetClock(); }
1545 void AdvanceTime(TimeDelta duration) {
1546 time_controller_.AdvanceTime(duration);
1547 }
1548
1549 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1550
1551 protected:
1552 virtual TaskQueueFactory* GetTaskQueueFactory() {
1553 return time_controller_.GetTaskQueueFactory();
1554 }
1555
1556 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001557 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001558 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001559 int codec_width_;
1560 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001561 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001562 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001563 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001564 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001565 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001566 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001567 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001568 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001569};
1570
mflodmancc3d4422017-08-03 08:27:51 -07001571TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001573 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001574 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001575 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001576 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001577 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001578 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001579}
1580
mflodmancc3d4422017-08-03 08:27:51 -07001581TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001582 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001583 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001584 // The encoder will cache up to one frame for a short duration. Adding two
1585 // frames means that the first frame will be dropped and the second frame will
1586 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001587 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001588 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001589 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001590 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001591 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001592
Henrik Boström381d1092020-05-12 18:49:07 +02001593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001594 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001595
Sebastian Janssona3177052018-04-10 13:05:49 +02001596 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001597 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001598 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1599
1600 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001601 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001602}
1603
mflodmancc3d4422017-08-03 08:27:51 -07001604TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001606 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001607 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001608 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001609
Henrik Boström381d1092020-05-12 18:49:07 +02001610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001611 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1612
Sebastian Janssona3177052018-04-10 13:05:49 +02001613 // The encoder will cache up to one frame for a short duration. Adding two
1614 // frames means that the first frame will be dropped and the second frame will
1615 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001616 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001617 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001618
Henrik Boström381d1092020-05-12 18:49:07 +02001619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001620 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001621 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001622 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1623 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001624 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001625}
1626
mflodmancc3d4422017-08-03 08:27:51 -07001627TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001629 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001630 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001631 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001632
1633 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001634 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001635
perkja49cbd32016-09-16 07:53:41 -07001636 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001637 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001638 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001639}
1640
mflodmancc3d4422017-08-03 08:27:51 -07001641TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001642 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001643 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001644
perkja49cbd32016-09-16 07:53:41 -07001645 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001646 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001647
mflodmancc3d4422017-08-03 08:27:51 -07001648 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001649 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001650 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001651 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1652 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001653}
1654
Markus Handell9a478b52021-11-18 16:07:01 +01001655TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1656 test::FrameForwarder source;
1657 video_stream_encoder_->SetSource(&source,
1658 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001660 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001661
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001662 int dropped_count = 0;
1663 stats_proxy_->SetDroppedFrameCallback(
1664 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1665 ++dropped_count;
1666 });
1667
Markus Handell9a478b52021-11-18 16:07:01 +01001668 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1669 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1670 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001671 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001672 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001673}
1674
Henrik Boström56db9ff2021-03-24 09:06:45 +01001675TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001677 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001678
1679 rtc::Event frame_destroyed_event;
1680 video_source_.IncomingCapturedFrame(
1681 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001682 WaitForEncodedFrame(1);
1683 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1684 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001685 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1686 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001687 video_stream_encoder_->Stop();
1688}
1689
Henrik Boström56db9ff2021-03-24 09:06:45 +01001690TEST_F(VideoStreamEncoderTest,
1691 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001692 // Use the cropping factory.
1693 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001694 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001695 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1696 kMaxPayloadLength);
1697 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1698
1699 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001700 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001701 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001702 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1703 WaitForEncodedFrame(1);
1704 // The encoder will have been configured once.
1705 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001706 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1707 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001708
1709 // Now send in a fake frame that needs to be cropped as the width/height
1710 // aren't divisible by 4 (see CreateEncoderStreams above).
1711 rtc::Event frame_destroyed_event;
1712 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1713 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001714 WaitForEncodedFrame(2);
1715 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1716 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001717 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1718 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001719 video_stream_encoder_->Stop();
1720}
1721
Evan Shrubsole895556e2020-10-05 09:15:13 +02001722TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001725
1726 video_source_.IncomingCapturedFrame(
1727 CreateNV12Frame(1, codec_width_, codec_height_));
1728 WaitForEncodedFrame(1);
1729 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1730 fake_encoder_.GetLastInputPixelFormat());
1731 video_stream_encoder_->Stop();
1732}
1733
Henrik Boström56db9ff2021-03-24 09:06:45 +01001734TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001736 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001737
1738 fake_encoder_.SetPreferredPixelFormats({});
1739
1740 rtc::Event frame_destroyed_event;
1741 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1742 1, &frame_destroyed_event, codec_width_, codec_height_));
1743 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001744 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001745 fake_encoder_.GetLastInputPixelFormat());
1746 video_stream_encoder_->Stop();
1747}
1748
Henrik Boström56db9ff2021-03-24 09:06:45 +01001749TEST_F(VideoStreamEncoderTest,
1750 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001753
1754 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1755
1756 rtc::Event frame_destroyed_event;
1757 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1758 1, &frame_destroyed_event, codec_width_, codec_height_));
1759 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001760 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001761 fake_encoder_.GetLastInputPixelFormat());
1762 video_stream_encoder_->Stop();
1763}
1764
Henrik Boström56db9ff2021-03-24 09:06:45 +01001765TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001768
1769 // Fake NV12 native frame does not allow mapping to I444.
1770 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1771
1772 rtc::Event frame_destroyed_event;
1773 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1774 1, &frame_destroyed_event, codec_width_, codec_height_));
1775 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001776 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001777 fake_encoder_.GetLastInputPixelFormat());
1778 video_stream_encoder_->Stop();
1779}
1780
Henrik Boström56db9ff2021-03-24 09:06:45 +01001781TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001783 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001784
1785 rtc::Event frame_destroyed_event;
1786 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1787 1, &frame_destroyed_event, codec_width_, codec_height_));
1788 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001789 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001790 fake_encoder_.GetLastInputPixelFormat());
1791 video_stream_encoder_->Stop();
1792}
1793
Ying Wang9b881ab2020-02-07 14:29:32 +01001794TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001797 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1798 WaitForEncodedFrame(1);
1799
Henrik Boström381d1092020-05-12 18:49:07 +02001800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001802 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1803 // frames. Adding two frames means that the first frame will be dropped and
1804 // the second frame will be sent to the encoder.
1805 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1806 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1807 WaitForEncodedFrame(3);
1808 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1809 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1810 WaitForEncodedFrame(5);
1811 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1812 video_stream_encoder_->Stop();
1813}
1814
mflodmancc3d4422017-08-03 08:27:51 -07001815TEST_F(VideoStreamEncoderTest,
1816 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001818 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001819 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001820
1821 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001822 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001823 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001824 // The encoder will have been configured once when the first frame is
1825 // received.
1826 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001827
1828 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001829 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001830 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001831 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001832 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001833
1834 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001835 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001836 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001837 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001838 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001839
mflodmancc3d4422017-08-03 08:27:51 -07001840 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001841}
1842
mflodmancc3d4422017-08-03 08:27:51 -07001843TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001844 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001845 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001846
1847 // Capture a frame and wait for it to synchronize with the encoder thread.
1848 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001849 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001850 // The encoder will have been configured once.
1851 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001852 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1853 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001854
1855 codec_width_ *= 2;
1856 codec_height_ *= 2;
1857 // Capture a frame with a higher resolution and wait for it to synchronize
1858 // with the encoder thread.
1859 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001860 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001861 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1862 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001863 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001864
mflodmancc3d4422017-08-03 08:27:51 -07001865 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001866}
1867
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001868TEST_F(VideoStreamEncoderTest,
1869 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001871 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001872
1873 // Capture a frame and wait for it to synchronize with the encoder thread.
1874 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1875 WaitForEncodedFrame(1);
1876
1877 VideoEncoderConfig video_encoder_config;
1878 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1879 // Changing the max payload data length recreates encoder.
1880 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1881 kMaxPayloadLength / 2);
1882
1883 // Capture a frame and wait for it to synchronize with the encoder thread.
1884 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1885 WaitForEncodedFrame(2);
1886 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1887
1888 video_stream_encoder_->Stop();
1889}
1890
Sergey Silkin5ee69672019-07-02 14:18:34 +02001891TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001893 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001894
1895 VideoEncoderConfig video_encoder_config;
1896 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001897 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1898 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001899 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1900 kMaxPayloadLength);
1901
1902 // Capture a frame and wait for it to synchronize with the encoder thread.
1903 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1904 WaitForEncodedFrame(1);
1905 // The encoder will have been configured once when the first frame is
1906 // received.
1907 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001908 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001909 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001910 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001911 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1912
Sergey Silkin6456e352019-07-08 17:56:40 +02001913 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1914 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001915 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1916 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001917 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1918 kMaxPayloadLength);
1919
1920 // Capture a frame and wait for it to synchronize with the encoder thread.
1921 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1922 WaitForEncodedFrame(2);
1923 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1924 // Bitrate limits have changed - rate allocator should be reconfigured,
1925 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001926 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001927 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001928 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001929 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001930 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001931
1932 video_stream_encoder_->Stop();
1933}
1934
Sergey Silkin6456e352019-07-08 17:56:40 +02001935TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001936 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001938 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001939
Sergey Silkincd02eba2020-01-20 14:48:40 +01001940 const uint32_t kMinEncBitrateKbps = 100;
1941 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001943 /*frame_size_pixels=*/codec_width_ * codec_height_,
1944 /*min_start_bitrate_bps=*/0,
1945 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1946 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001947 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1948
Sergey Silkincd02eba2020-01-20 14:48:40 +01001949 VideoEncoderConfig video_encoder_config;
1950 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1951 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1952 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1953 (kMinEncBitrateKbps + 1) * 1000;
1954 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1955 kMaxPayloadLength);
1956
1957 // When both encoder and app provide bitrate limits, the intersection of
1958 // provided sets should be used.
1959 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1960 WaitForEncodedFrame(1);
1961 EXPECT_EQ(kMaxEncBitrateKbps,
1962 bitrate_allocator_factory_.codec_config().maxBitrate);
1963 EXPECT_EQ(kMinEncBitrateKbps + 1,
1964 bitrate_allocator_factory_.codec_config().minBitrate);
1965
1966 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1967 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1968 (kMinEncBitrateKbps - 1) * 1000;
1969 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1970 kMaxPayloadLength);
1971 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001972 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001973 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001974 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001975 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001976 bitrate_allocator_factory_.codec_config().minBitrate);
1977
Sergey Silkincd02eba2020-01-20 14:48:40 +01001978 video_stream_encoder_->Stop();
1979}
1980
1981TEST_F(VideoStreamEncoderTest,
1982 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001984 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001985
1986 const uint32_t kMinAppBitrateKbps = 100;
1987 const uint32_t kMaxAppBitrateKbps = 200;
1988 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1989 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1990 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1991 /*frame_size_pixels=*/codec_width_ * codec_height_,
1992 /*min_start_bitrate_bps=*/0,
1993 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1994 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1995 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1996
1997 VideoEncoderConfig video_encoder_config;
1998 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1999 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2000 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2001 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002002 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2003 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004
Sergey Silkincd02eba2020-01-20 14:48:40 +01002005 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2006 WaitForEncodedFrame(1);
2007 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002008 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002009 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002010 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002011
2012 video_stream_encoder_->Stop();
2013}
2014
2015TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002016 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002018 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002019
2020 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002021 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002022 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002024 fake_encoder_.SetResolutionBitrateLimits(
2025 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2026
2027 VideoEncoderConfig video_encoder_config;
2028 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2029 video_encoder_config.max_bitrate_bps = 0;
2030 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2031 kMaxPayloadLength);
2032
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002033 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002034 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2035 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002036 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2037 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002038 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2039 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2040
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002041 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002042 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2043 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002044 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2045 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002046 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2047 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2048
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002049 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002050 // encoder for 360p should be used.
2051 video_source_.IncomingCapturedFrame(
2052 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2053 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002054 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2055 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002056 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2057 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2058
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002060 // ignored.
2061 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2062 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002063 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2064 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002065 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2066 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002067 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2068 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002069 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2070 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2071
2072 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2073 // for 270p should be used.
2074 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2075 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002076 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2077 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002078 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2079 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2080
2081 video_stream_encoder_->Stop();
2082}
2083
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002084TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002086 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002087
2088 VideoEncoderConfig video_encoder_config;
2089 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2090 video_encoder_config.max_bitrate_bps = 0;
2091 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2092 kMaxPayloadLength);
2093
2094 // Encode 720p frame to get the default encoder target bitrate.
2095 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2096 WaitForEncodedFrame(1);
2097 const uint32_t kDefaultTargetBitrateFor720pKbps =
2098 bitrate_allocator_factory_.codec_config()
2099 .simulcastStream[0]
2100 .targetBitrate;
2101
2102 // Set the max recommended encoder bitrate to something lower than the default
2103 // target bitrate.
2104 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2105 1280 * 720, 10 * 1000, 10 * 1000,
2106 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2107 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2108
2109 // Change resolution to trigger encoder reinitialization.
2110 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2111 WaitForEncodedFrame(2);
2112 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2113 WaitForEncodedFrame(3);
2114
2115 // Ensure the target bitrate is capped by the max bitrate.
2116 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2117 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2118 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2119 .simulcastStream[0]
2120 .targetBitrate *
2121 1000,
2122 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2123
2124 video_stream_encoder_->Stop();
2125}
2126
Åsa Perssona7e34d32021-01-20 15:36:13 +01002127TEST_F(VideoStreamEncoderTest,
2128 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2129 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2130 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2131 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2132 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2133 fake_encoder_.SetResolutionBitrateLimits(
2134 {kEncoderLimits270p, kEncoderLimits360p});
2135
2136 // Two streams, highest stream active.
2137 VideoEncoderConfig config;
2138 const int kNumStreams = 2;
2139 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2140 config.max_bitrate_bps = 0;
2141 config.simulcast_layers[0].active = false;
2142 config.simulcast_layers[1].active = true;
2143 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002144 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002145 "VP8", /*max qp*/ 56, /*screencast*/ false,
2146 /*screenshare enabled*/ false);
2147 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2148
2149 // The encoder bitrate limits for 270p should be used.
2150 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2151 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002152 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002153 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002154 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002155 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002156 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002157
2158 // The encoder bitrate limits for 360p should be used.
2159 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2160 EXPECT_FALSE(WaitForFrame(1000));
2161 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002162 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002163 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002164 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002165
2166 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2167 video_source_.IncomingCapturedFrame(
2168 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2169 EXPECT_FALSE(WaitForFrame(1000));
2170 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002171 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002172 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002173 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002174
2175 // Resolution higher than 360p. Encoder limits should be ignored.
2176 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2177 EXPECT_FALSE(WaitForFrame(1000));
2178 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002179 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002180 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002181 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002182 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002183 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002184 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002185 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002186
2187 // Resolution lower than 270p. The encoder limits for 270p should be used.
2188 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002191 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002192 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002193 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002194
2195 video_stream_encoder_->Stop();
2196}
2197
2198TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002199 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2200 // Two streams, highest stream active.
2201 VideoEncoderConfig config;
2202 const int kNumStreams = 2;
2203 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2204 config.max_bitrate_bps = 0;
2205 config.simulcast_layers[0].active = false;
2206 config.simulcast_layers[1].active = true;
2207 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002208 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002209 "VP8", /*max qp*/ 56, /*screencast*/ false,
2210 /*screenshare enabled*/ false);
2211 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2212
2213 // Default bitrate limits for 270p should be used.
2214 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2215 kDefaultLimits270p =
2216 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002217 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002218 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2219 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002220 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002221 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002223 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002225
2226 // Default bitrate limits for 360p should be used.
2227 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2228 kDefaultLimits360p =
2229 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002230 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002231 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2232 EXPECT_FALSE(WaitForFrame(1000));
2233 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002235 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002237
2238 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2239 video_source_.IncomingCapturedFrame(
2240 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2241 EXPECT_FALSE(WaitForFrame(1000));
2242 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002243 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002244 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002245 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002246
2247 // Default bitrate limits for 540p should be used.
2248 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2249 kDefaultLimits540p =
2250 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002251 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002252 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2253 EXPECT_FALSE(WaitForFrame(1000));
2254 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002255 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002256 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002257 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002258
2259 video_stream_encoder_->Stop();
2260}
2261
2262TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002263 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2264 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2265 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2266 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2267 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2268 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2269 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2270 fake_encoder_.SetResolutionBitrateLimits(
2271 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2272
2273 // Three streams, middle stream active.
2274 VideoEncoderConfig config;
2275 const int kNumStreams = 3;
2276 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2277 config.simulcast_layers[0].active = false;
2278 config.simulcast_layers[1].active = true;
2279 config.simulcast_layers[2].active = false;
2280 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002281 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002282 "VP8", /*max qp*/ 56, /*screencast*/ false,
2283 /*screenshare enabled*/ false);
2284 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2285
2286 // The encoder bitrate limits for 360p should be used.
2287 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2288 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002289 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002290 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002291 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002292 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002293 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002294
2295 // The encoder bitrate limits for 270p should be used.
2296 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2297 EXPECT_FALSE(WaitForFrame(1000));
2298 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002299 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002300 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002301 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002302
2303 video_stream_encoder_->Stop();
2304}
2305
2306TEST_F(VideoStreamEncoderTest,
2307 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2308 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2309 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2310 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2311 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2312 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2313 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2314 fake_encoder_.SetResolutionBitrateLimits(
2315 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2316
2317 // Three streams, lowest stream active.
2318 VideoEncoderConfig config;
2319 const int kNumStreams = 3;
2320 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2321 config.simulcast_layers[0].active = true;
2322 config.simulcast_layers[1].active = false;
2323 config.simulcast_layers[2].active = false;
2324 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002325 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002326 "VP8", /*max qp*/ 56, /*screencast*/ false,
2327 /*screenshare enabled*/ false);
2328 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2329
2330 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2331 // on lowest stream, limits for 270p should not be used
2332 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2333 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002334 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002335 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002336 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002337 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002338 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002339
2340 video_stream_encoder_->Stop();
2341}
2342
2343TEST_F(VideoStreamEncoderTest,
2344 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2345 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2346 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2347 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2348 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2349 fake_encoder_.SetResolutionBitrateLimits(
2350 {kEncoderLimits270p, kEncoderLimits360p});
2351 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2352
2353 // Two streams, highest stream active.
2354 VideoEncoderConfig config;
2355 const int kNumStreams = 2;
2356 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2357 config.simulcast_layers[0].active = false;
2358 config.simulcast_layers[1].active = true;
2359 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2360 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002361 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002362 "VP8", /*max qp*/ 56, /*screencast*/ false,
2363 /*screenshare enabled*/ false);
2364 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2365
2366 // The encoder bitrate limits for 270p should be used.
2367 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2368 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002369 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002370 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002371 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002372 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002373 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002374
2375 // The max configured bitrate is less than the encoder limit for 360p.
2376 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2377 EXPECT_FALSE(WaitForFrame(1000));
2378 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002379 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002380 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002381 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002382
2383 video_stream_encoder_->Stop();
2384}
2385
mflodmancc3d4422017-08-03 08:27:51 -07002386TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002387 EXPECT_TRUE(video_source_.has_sinks());
2388 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002389 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002390 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002391 EXPECT_FALSE(video_source_.has_sinks());
2392 EXPECT_TRUE(new_video_source.has_sinks());
2393
mflodmancc3d4422017-08-03 08:27:51 -07002394 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002395}
2396
mflodmancc3d4422017-08-03 08:27:51 -07002397TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002398 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002399 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002400 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002401 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002402}
2403
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002404class ResolutionAlignmentTest
2405 : public VideoStreamEncoderTest,
2406 public ::testing::WithParamInterface<
2407 ::testing::tuple<int, std::vector<double>>> {
2408 public:
2409 ResolutionAlignmentTest()
2410 : requested_alignment_(::testing::get<0>(GetParam())),
2411 scale_factors_(::testing::get<1>(GetParam())) {}
2412
2413 protected:
2414 const int requested_alignment_;
2415 const std::vector<double> scale_factors_;
2416};
2417
2418INSTANTIATE_TEST_SUITE_P(
2419 AlignmentAndScaleFactors,
2420 ResolutionAlignmentTest,
2421 ::testing::Combine(
2422 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2423 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2424 std::vector<double>{-1.0, -1.0},
2425 std::vector<double>{-1.0, -1.0, -1.0},
2426 std::vector<double>{4.0, 2.0, 1.0},
2427 std::vector<double>{9999.0, -1.0, 1.0},
2428 std::vector<double>{3.99, 2.01, 1.0},
2429 std::vector<double>{4.9, 1.7, 1.25},
2430 std::vector<double>{10.0, 4.0, 3.0},
2431 std::vector<double>{1.75, 3.5},
2432 std::vector<double>{1.5, 2.5},
2433 std::vector<double>{1.3, 1.0})));
2434
2435TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2436 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002437 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002438 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2439 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2440
2441 // Fill config with the scaling factor by which to reduce encoding size.
2442 const int num_streams = scale_factors_.size();
2443 VideoEncoderConfig config;
2444 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2445 for (int i = 0; i < num_streams; ++i) {
2446 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2447 }
2448 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002449 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002450 "VP8", /*max qp*/ 56, /*screencast*/ false,
2451 /*screenshare enabled*/ false);
2452 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2453
Henrik Boström381d1092020-05-12 18:49:07 +02002454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002455 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2456 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002457 // Wait for all layers before triggering event.
2458 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002459
2460 // On the 1st frame, we should have initialized the encoder and
2461 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002462 int64_t timestamp_ms = kFrameIntervalMs;
2463 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2464 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002465 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002466
2467 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2468 // (It's up the to the encoder to potentially drop the previous frame,
2469 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002470 timestamp_ms += kFrameIntervalMs;
2471 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2472 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002473 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002474
Asa Persson606d3cb2021-10-04 10:07:11 +02002475 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002476 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2477 // Frame size should be a multiple of the requested alignment.
2478 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2479 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2480 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2481 // Aspect ratio should match.
2482 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2483 codec.height * codec.simulcastStream[i].width);
2484 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002485
2486 video_stream_encoder_->Stop();
2487}
2488
Jonathan Yubc771b72017-12-08 17:04:29 -08002489TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2490 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002491 const int kWidth = 1280;
2492 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002493
2494 // We rely on the automatic resolution adaptation, but we handle framerate
2495 // adaptation manually by mocking the stats proxy.
2496 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002497
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002498 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002500 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002501 video_stream_encoder_->SetSource(&video_source_,
2502 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002503 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002504 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002505 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002506 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2507
Jonathan Yubc771b72017-12-08 17:04:29 -08002508 // Adapt down as far as possible.
2509 rtc::VideoSinkWants last_wants;
2510 int64_t t = 1;
2511 int loop_count = 0;
2512 do {
2513 ++loop_count;
2514 last_wants = video_source_.sink_wants();
2515
2516 // Simulate the framerate we've been asked to adapt to.
2517 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2518 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2519 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2520 mock_stats.input_frame_rate = fps;
2521 stats_proxy_->SetMockStats(mock_stats);
2522
2523 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2524 sink_.WaitForEncodedFrame(t);
2525 t += frame_interval_ms;
2526
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002528 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002529 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002530 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2531 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002532 } while (video_source_.sink_wants().max_pixel_count <
2533 last_wants.max_pixel_count ||
2534 video_source_.sink_wants().max_framerate_fps <
2535 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002536
Jonathan Yubc771b72017-12-08 17:04:29 -08002537 // Verify that we've adapted all the way down.
2538 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002539 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002540 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2541 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002542 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002543 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2544 *video_source_.last_sent_height());
2545 EXPECT_EQ(kMinBalancedFramerateFps,
2546 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002547
Jonathan Yubc771b72017-12-08 17:04:29 -08002548 // Adapt back up the same number of times we adapted down.
2549 for (int i = 0; i < loop_count - 1; ++i) {
2550 last_wants = video_source_.sink_wants();
2551
2552 // Simulate the framerate we've been asked to adapt to.
2553 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2554 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2555 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2556 mock_stats.input_frame_rate = fps;
2557 stats_proxy_->SetMockStats(mock_stats);
2558
2559 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2560 sink_.WaitForEncodedFrame(t);
2561 t += frame_interval_ms;
2562
Henrik Boström91aa7322020-04-28 12:24:33 +02002563 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002564 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002565 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002566 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2567 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002568 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2569 last_wants.max_pixel_count ||
2570 video_source_.sink_wants().max_framerate_fps >
2571 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002572 }
2573
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002574 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002575 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002576 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002577 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2578 EXPECT_EQ((loop_count - 1) * 2,
2579 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002580
mflodmancc3d4422017-08-03 08:27:51 -07002581 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002582}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002583
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002584TEST_F(VideoStreamEncoderTest,
2585 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002586 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2587 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002588 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002589
2590 const int kFrameWidth = 1280;
2591 const int kFrameHeight = 720;
2592
2593 int64_t ntp_time = kFrameIntervalMs;
2594
2595 // Force an input frame rate to be available, or the adaptation call won't
2596 // know what framerate to adapt form.
2597 const int kInputFps = 30;
2598 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2599 stats.input_frame_rate = kInputFps;
2600 stats_proxy_->SetMockStats(stats);
2601
2602 video_source_.set_adaptation_enabled(true);
2603 video_stream_encoder_->SetSource(
2604 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002605 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002606 video_source_.IncomingCapturedFrame(
2607 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2608 sink_.WaitForEncodedFrame(ntp_time);
2609 ntp_time += kFrameIntervalMs;
2610
2611 // Trigger CPU overuse.
2612 video_stream_encoder_->TriggerCpuOveruse();
2613 video_source_.IncomingCapturedFrame(
2614 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2615 sink_.WaitForEncodedFrame(ntp_time);
2616 ntp_time += kFrameIntervalMs;
2617
2618 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2619 EXPECT_EQ(std::numeric_limits<int>::max(),
2620 video_source_.sink_wants().max_pixel_count);
2621 // Some framerate constraint should be set.
2622 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2623 EXPECT_LT(restricted_fps, kInputFps);
2624 video_source_.IncomingCapturedFrame(
2625 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2626 sink_.WaitForEncodedFrame(ntp_time);
2627 ntp_time += 100;
2628
Henrik Boström2671dac2020-05-19 16:29:09 +02002629 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002630 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2631 // Give the encoder queue time to process the change in degradation preference
2632 // by waiting for an encoded frame.
2633 video_source_.IncomingCapturedFrame(
2634 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2635 sink_.WaitForEncodedFrame(ntp_time);
2636 ntp_time += kFrameIntervalMs;
2637
2638 video_stream_encoder_->TriggerQualityLow();
2639 video_source_.IncomingCapturedFrame(
2640 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2641 sink_.WaitForEncodedFrame(ntp_time);
2642 ntp_time += kFrameIntervalMs;
2643
2644 // Some resolution constraint should be set.
2645 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2646 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2647 kFrameWidth * kFrameHeight);
2648 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2649
2650 int pixel_count = video_source_.sink_wants().max_pixel_count;
2651 // Triggering a CPU underuse should not change the sink wants since it has
2652 // not been overused for resolution since we changed degradation preference.
2653 video_stream_encoder_->TriggerCpuUnderuse();
2654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2656 sink_.WaitForEncodedFrame(ntp_time);
2657 ntp_time += kFrameIntervalMs;
2658 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2659 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2660
Evan Shrubsole64469032020-06-11 10:45:29 +02002661 // Change the degradation preference back. CPU underuse should not adapt since
2662 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002663 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002664 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2665 video_source_.IncomingCapturedFrame(
2666 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2667 sink_.WaitForEncodedFrame(ntp_time);
2668 ntp_time += 100;
2669 // Resolution adaptations is gone after changing degradation preference.
2670 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2671 EXPECT_EQ(std::numeric_limits<int>::max(),
2672 video_source_.sink_wants().max_pixel_count);
2673 // The fps adaptation from above is now back.
2674 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2675
2676 // Trigger CPU underuse.
2677 video_stream_encoder_->TriggerCpuUnderuse();
2678 video_source_.IncomingCapturedFrame(
2679 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2680 sink_.WaitForEncodedFrame(ntp_time);
2681 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002682 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2683
2684 // Trigger QP underuse, fps should return to normal.
2685 video_stream_encoder_->TriggerQualityHigh();
2686 video_source_.IncomingCapturedFrame(
2687 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2688 sink_.WaitForEncodedFrame(ntp_time);
2689 ntp_time += kFrameIntervalMs;
2690 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002691
2692 video_stream_encoder_->Stop();
2693}
2694
mflodmancc3d4422017-08-03 08:27:51 -07002695TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002698 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002699
sprangc5d62e22017-04-02 23:53:04 -07002700 const int kFrameWidth = 1280;
2701 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002702
Åsa Persson8c1bf952018-09-13 10:42:19 +02002703 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002704
kthelgason5e13d412016-12-01 03:59:51 -08002705 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002706 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002707 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002708 frame_timestamp += kFrameIntervalMs;
2709
perkj803d97f2016-11-01 11:45:46 -07002710 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002711 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002712 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002713 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002714 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002715 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002716
asapersson0944a802017-04-07 00:57:58 -07002717 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002718 // wanted resolution.
2719 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2720 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2721 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002722 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002723
2724 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002725 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002726 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002727 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002728 // Give the encoder queue time to process the change in degradation preference
2729 // by waiting for an encoded frame.
2730 new_video_source.IncomingCapturedFrame(
2731 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2732 sink_.WaitForEncodedFrame(frame_timestamp);
2733 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002734 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002735 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002736
sprangc5d62e22017-04-02 23:53:04 -07002737 // Force an input frame rate to be available, or the adaptation call won't
2738 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002739 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002740 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002741 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002742 stats_proxy_->SetMockStats(stats);
2743
mflodmancc3d4422017-08-03 08:27:51 -07002744 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002745 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002746 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002747 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002748 frame_timestamp += kFrameIntervalMs;
2749
2750 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002751 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002752 EXPECT_EQ(std::numeric_limits<int>::max(),
2753 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002754 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002755
asapersson02465b82017-04-10 01:12:52 -07002756 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002757 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2758 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002759 // Give the encoder queue time to process the change in degradation preference
2760 // by waiting for an encoded frame.
2761 new_video_source.IncomingCapturedFrame(
2762 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2763 sink_.WaitForEncodedFrame(frame_timestamp);
2764 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002765 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002766
mflodmancc3d4422017-08-03 08:27:51 -07002767 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002768 new_video_source.IncomingCapturedFrame(
2769 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002771 frame_timestamp += kFrameIntervalMs;
2772
2773 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002774 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002775
2776 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002777 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002778 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002779 // Give the encoder queue time to process the change in degradation preference
2780 // by waiting for an encoded frame.
2781 new_video_source.IncomingCapturedFrame(
2782 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2783 sink_.WaitForEncodedFrame(frame_timestamp);
2784 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002785 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2786 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002787 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002788 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002789
2790 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002791 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002792 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002793 // Give the encoder queue time to process the change in degradation preference
2794 // by waiting for an encoded frame.
2795 new_video_source.IncomingCapturedFrame(
2796 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2797 sink_.WaitForEncodedFrame(frame_timestamp);
2798 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002799 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2800 EXPECT_EQ(std::numeric_limits<int>::max(),
2801 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002802 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002803
mflodmancc3d4422017-08-03 08:27:51 -07002804 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002805}
2806
mflodmancc3d4422017-08-03 08:27:51 -07002807TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002809 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002810
asaperssonfab67072017-04-04 05:51:49 -07002811 const int kWidth = 1280;
2812 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002813 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002814 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002815 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2816 EXPECT_FALSE(stats.bw_limited_resolution);
2817 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2818
2819 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002820 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002821 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002822 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002823
2824 stats = stats_proxy_->GetStats();
2825 EXPECT_TRUE(stats.bw_limited_resolution);
2826 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2827
2828 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002829 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002830 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002831 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002832
2833 stats = stats_proxy_->GetStats();
2834 EXPECT_FALSE(stats.bw_limited_resolution);
2835 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2836 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2837
mflodmancc3d4422017-08-03 08:27:51 -07002838 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002839}
2840
mflodmancc3d4422017-08-03 08:27:51 -07002841TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002842 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002843 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002844
2845 const int kWidth = 1280;
2846 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002847 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002848 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002849 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2850 EXPECT_FALSE(stats.cpu_limited_resolution);
2851 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2852
2853 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002854 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002855 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002856 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002857
2858 stats = stats_proxy_->GetStats();
2859 EXPECT_TRUE(stats.cpu_limited_resolution);
2860 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2861
2862 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002863 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002864 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002865 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002866
2867 stats = stats_proxy_->GetStats();
2868 EXPECT_FALSE(stats.cpu_limited_resolution);
2869 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002870 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002871
mflodmancc3d4422017-08-03 08:27:51 -07002872 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002873}
2874
mflodmancc3d4422017-08-03 08:27:51 -07002875TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002877 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002878
asaperssonfab67072017-04-04 05:51:49 -07002879 const int kWidth = 1280;
2880 const int kHeight = 720;
2881 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002883 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002884 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002885 EXPECT_FALSE(stats.cpu_limited_resolution);
2886 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2887
asaperssonfab67072017-04-04 05:51:49 -07002888 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002889 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002890 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002891 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002892 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002893 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002894 EXPECT_TRUE(stats.cpu_limited_resolution);
2895 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2896
2897 // Set new source with adaptation still enabled.
2898 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002899 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002900 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002901
asaperssonfab67072017-04-04 05:51:49 -07002902 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002904 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002905 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002906 EXPECT_TRUE(stats.cpu_limited_resolution);
2907 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2908
2909 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002910 video_stream_encoder_->SetSource(&new_video_source,
2911 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002912
asaperssonfab67072017-04-04 05:51:49 -07002913 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002914 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002915 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002916 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002917 EXPECT_FALSE(stats.cpu_limited_resolution);
2918 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2919
2920 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002921 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002922 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002923
asaperssonfab67072017-04-04 05:51:49 -07002924 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002925 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002926 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002927 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002928 EXPECT_TRUE(stats.cpu_limited_resolution);
2929 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2930
asaperssonfab67072017-04-04 05:51:49 -07002931 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002932 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002933 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002934 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002935 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002936 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002937 EXPECT_FALSE(stats.cpu_limited_resolution);
2938 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002939 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002940
mflodmancc3d4422017-08-03 08:27:51 -07002941 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002942}
2943
mflodmancc3d4422017-08-03 08:27:51 -07002944TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002946 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002947
asaperssonfab67072017-04-04 05:51:49 -07002948 const int kWidth = 1280;
2949 const int kHeight = 720;
2950 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002951 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002952 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002953 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002954 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002955 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002956
2957 // Set new source with adaptation still enabled.
2958 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002959 video_stream_encoder_->SetSource(&new_video_source,
2960 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002961
asaperssonfab67072017-04-04 05:51:49 -07002962 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002963 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002964 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002965 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002966 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002967 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002968
asaperssonfab67072017-04-04 05:51:49 -07002969 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002970 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002971 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002972 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002973 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002974 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002975 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002976 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002977
asaperssonfab67072017-04-04 05:51:49 -07002978 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002979 video_stream_encoder_->SetSource(&new_video_source,
2980 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002981
asaperssonfab67072017-04-04 05:51:49 -07002982 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002983 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002984 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002985 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002986 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002987 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002988
asapersson02465b82017-04-10 01:12:52 -07002989 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002990 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002991 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002992
asaperssonfab67072017-04-04 05:51:49 -07002993 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002994 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002995 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002996 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002997 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002998 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2999 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003000
mflodmancc3d4422017-08-03 08:27:51 -07003001 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003002}
3003
mflodmancc3d4422017-08-03 08:27:51 -07003004TEST_F(VideoStreamEncoderTest,
3005 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003007 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003008
3009 const int kWidth = 1280;
3010 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003011 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003012 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003013 video_source_.IncomingCapturedFrame(
3014 CreateFrame(timestamp_ms, kWidth, kHeight));
3015 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003016 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3018 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3019
3020 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003021 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003022 timestamp_ms += kFrameIntervalMs;
3023 video_source_.IncomingCapturedFrame(
3024 CreateFrame(timestamp_ms, kWidth, kHeight));
3025 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003026 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3027 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3029
3030 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003031 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003032 timestamp_ms += kFrameIntervalMs;
3033 video_source_.IncomingCapturedFrame(
3034 CreateFrame(timestamp_ms, kWidth, kHeight));
3035 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003036 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3037 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3038 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3039
Niels Möller4db138e2018-04-19 09:04:13 +02003040 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003041 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003042
3043 VideoEncoderConfig video_encoder_config;
3044 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3045 // Make format different, to force recreation of encoder.
3046 video_encoder_config.video_format.parameters["foo"] = "foo";
3047 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003048 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003049 timestamp_ms += kFrameIntervalMs;
3050 video_source_.IncomingCapturedFrame(
3051 CreateFrame(timestamp_ms, kWidth, kHeight));
3052 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003053 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3054 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3055 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3056
mflodmancc3d4422017-08-03 08:27:51 -07003057 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003058}
3059
mflodmancc3d4422017-08-03 08:27:51 -07003060TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003061 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003063 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003064
3065 const int kWidth = 1280;
3066 const int kHeight = 720;
3067 int sequence = 1;
3068
3069 // Enable BALANCED preference, no initial limitation.
3070 test::FrameForwarder source;
3071 video_stream_encoder_->SetSource(&source,
3072 webrtc::DegradationPreference::BALANCED);
3073 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3074 WaitForEncodedFrame(sequence++);
3075 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3076 EXPECT_FALSE(stats.cpu_limited_resolution);
3077 EXPECT_FALSE(stats.cpu_limited_framerate);
3078 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3079
3080 // Trigger CPU overuse, should now adapt down.
3081 video_stream_encoder_->TriggerCpuOveruse();
3082 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3083 WaitForEncodedFrame(sequence++);
3084 stats = stats_proxy_->GetStats();
3085 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3086
3087 // Set new degradation preference should clear restrictions since we changed
3088 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003089 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003090 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3091 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3092 WaitForEncodedFrame(sequence++);
3093 stats = stats_proxy_->GetStats();
3094 EXPECT_FALSE(stats.cpu_limited_resolution);
3095 EXPECT_FALSE(stats.cpu_limited_framerate);
3096 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3097
3098 // Force an input frame rate to be available, or the adaptation call won't
3099 // know what framerate to adapt from.
3100 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3101 mock_stats.input_frame_rate = 30;
3102 stats_proxy_->SetMockStats(mock_stats);
3103 video_stream_encoder_->TriggerCpuOveruse();
3104 stats_proxy_->ResetMockStats();
3105 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3106 WaitForEncodedFrame(sequence++);
3107
3108 // We have now adapted once.
3109 stats = stats_proxy_->GetStats();
3110 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3111
3112 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003113 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3114 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003115 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3116 WaitForEncodedFrame(sequence++);
3117 stats = stats_proxy_->GetStats();
3118 EXPECT_FALSE(stats.cpu_limited_resolution);
3119 EXPECT_FALSE(stats.cpu_limited_framerate);
3120 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3121
3122 video_stream_encoder_->Stop();
3123}
3124
3125TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003126 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003128 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003129
asapersson0944a802017-04-07 00:57:58 -07003130 const int kWidth = 1280;
3131 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003132 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003133
asaperssonfab67072017-04-04 05:51:49 -07003134 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003135 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003136 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003137 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003138 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003139 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3140
asapersson02465b82017-04-10 01:12:52 -07003141 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003142 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003143 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003144 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003145 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003146 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003147 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003148 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3149
3150 // Set new source with adaptation still enabled.
3151 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003152 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003153 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003154
3155 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003156 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003157 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003158 stats = stats_proxy_->GetStats();
3159 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003160 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003161 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3162
sprangc5d62e22017-04-02 23:53:04 -07003163 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003165 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003166 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003167 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003168 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003169 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003170 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003171 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003172 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003173 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3174
sprangc5d62e22017-04-02 23:53:04 -07003175 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003176 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003177 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3178 mock_stats.input_frame_rate = 30;
3179 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003180 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003181 stats_proxy_->ResetMockStats();
3182
3183 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003184 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003185 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003186
3187 // Framerate now adapted.
3188 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003189 EXPECT_FALSE(stats.cpu_limited_resolution);
3190 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003191 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3192
3193 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003194 video_stream_encoder_->SetSource(&new_video_source,
3195 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003196 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003197 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003198 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003199
3200 stats = stats_proxy_->GetStats();
3201 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003202 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003203 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3204
3205 // Try to trigger overuse. Should not succeed.
3206 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003208 stats_proxy_->ResetMockStats();
3209
3210 stats = stats_proxy_->GetStats();
3211 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003212 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003213 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3214
3215 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003216 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003217 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003218 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003219 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003220 stats = stats_proxy_->GetStats();
3221 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003222 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003223 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003224
3225 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003226 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003227 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003228 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003229 stats = stats_proxy_->GetStats();
3230 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003231 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003232 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3233
3234 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003235 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003236 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003237 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003238 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003239 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003240 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003241 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003242 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003243 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003244 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3245
3246 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003247 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003248 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003249 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003250 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003251 stats = stats_proxy_->GetStats();
3252 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003253 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003254 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003255 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003256
mflodmancc3d4422017-08-03 08:27:51 -07003257 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003258}
3259
mflodmancc3d4422017-08-03 08:27:51 -07003260TEST_F(VideoStreamEncoderTest,
3261 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003262 const int kWidth = 1280;
3263 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003265 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003266
asaperssonfab67072017-04-04 05:51:49 -07003267 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003268 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003269
asaperssonfab67072017-04-04 05:51:49 -07003270 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003271 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003272
asaperssonfab67072017-04-04 05:51:49 -07003273 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003274 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003275
asaperssonfab67072017-04-04 05:51:49 -07003276 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003277 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003278
kthelgason876222f2016-11-29 01:44:11 -08003279 // Expect a scale down.
3280 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003281 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003282
asapersson02465b82017-04-10 01:12:52 -07003283 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003284 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003285 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003286 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003287
asaperssonfab67072017-04-04 05:51:49 -07003288 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003289 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003290 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003291 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003292
asaperssonfab67072017-04-04 05:51:49 -07003293 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003294 EXPECT_EQ(std::numeric_limits<int>::max(),
3295 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003296
asaperssonfab67072017-04-04 05:51:49 -07003297 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003298 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003299 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003300 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003301
asapersson02465b82017-04-10 01:12:52 -07003302 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003303 EXPECT_EQ(std::numeric_limits<int>::max(),
3304 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003305
mflodmancc3d4422017-08-03 08:27:51 -07003306 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003307}
3308
mflodmancc3d4422017-08-03 08:27:51 -07003309TEST_F(VideoStreamEncoderTest,
3310 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003311 const int kWidth = 1280;
3312 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003314 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003315
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003316 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003317 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003318 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003319 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003320
3321 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003322 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003323 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003324 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3325 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3326
3327 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003328 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003329 EXPECT_THAT(source.sink_wants(),
3330 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003331 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3332 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3333 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3334
3335 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003336 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003337 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3338 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3339 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3340
mflodmancc3d4422017-08-03 08:27:51 -07003341 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003342}
3343
mflodmancc3d4422017-08-03 08:27:51 -07003344TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003345 const int kWidth = 1280;
3346 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003349
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003350 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003351 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003352 video_stream_encoder_->SetSource(&source,
3353 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003354 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3355 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003356 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003357
3358 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003359 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003360 EXPECT_THAT(source.sink_wants(),
3361 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003362 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3363 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3364 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3365
3366 // Trigger adapt down for same input resolution, expect no change.
3367 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3368 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003369 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003370 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3372 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3373
3374 // Trigger adapt down for larger input resolution, expect no change.
3375 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3376 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003377 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003378 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3380 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3381
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003383}
3384
mflodmancc3d4422017-08-03 08:27:51 -07003385TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003386 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3387 const int kWidth = 640;
3388 const int kHeight = 360;
3389 const int64_t kFrameIntervalMs = 150;
3390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003391 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003392
3393 // Enable BALANCED preference, no initial limitation.
3394 AdaptingFrameForwarder source(&time_controller_);
3395 source.set_adaptation_enabled(true);
3396 video_stream_encoder_->SetSource(&source,
3397 webrtc::DegradationPreference::BALANCED);
3398
3399 int64_t timestamp_ms = kFrameIntervalMs;
3400 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3401 sink_.WaitForEncodedFrame(kWidth, kHeight);
3402 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3404 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
3407 // Trigger adapt down, expect reduced fps (640x360@15fps).
3408 video_stream_encoder_->TriggerQualityLow();
3409 timestamp_ms += kFrameIntervalMs;
3410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3411 sink_.WaitForEncodedFrame(timestamp_ms);
3412 EXPECT_THAT(source.sink_wants(),
3413 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3416 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3417
3418 // Source requests 270p, expect reduced resolution (480x270@15fps).
3419 source.OnOutputFormatRequest(480, 270);
3420 timestamp_ms += kFrameIntervalMs;
3421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3422 WaitForEncodedFrame(480, 270);
3423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
3425 // Trigger adapt down, expect reduced fps (480x270@10fps).
3426 video_stream_encoder_->TriggerQualityLow();
3427 timestamp_ms += kFrameIntervalMs;
3428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3429 sink_.WaitForEncodedFrame(timestamp_ms);
3430 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3431 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3432 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3433 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3434
3435 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3436 source.OnOutputFormatRequest(320, 180);
3437 timestamp_ms += kFrameIntervalMs;
3438 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3439 WaitForEncodedFrame(320, 180);
3440 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3441
3442 // Trigger adapt down, expect reduced fps (320x180@7fps).
3443 video_stream_encoder_->TriggerQualityLow();
3444 timestamp_ms += kFrameIntervalMs;
3445 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3446 sink_.WaitForEncodedFrame(timestamp_ms);
3447 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3448 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3449 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3450 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3451
3452 // Source requests VGA, expect increased resolution (640x360@7fps).
3453 source.OnOutputFormatRequest(640, 360);
3454 timestamp_ms += kFrameIntervalMs;
3455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3456 WaitForEncodedFrame(timestamp_ms);
3457 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3458
3459 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3460 video_stream_encoder_->TriggerQualityHigh();
3461 timestamp_ms += kFrameIntervalMs;
3462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3463 WaitForEncodedFrame(timestamp_ms);
3464 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3467 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3468
3469 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3470 video_stream_encoder_->TriggerQualityHigh();
3471 timestamp_ms += kFrameIntervalMs;
3472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3473 WaitForEncodedFrame(timestamp_ms);
3474 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3475 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3476 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3477 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3478
3479 // Trigger adapt up, expect increased fps (640x360@maxfps).
3480 video_stream_encoder_->TriggerQualityHigh();
3481 timestamp_ms += kFrameIntervalMs;
3482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3483 WaitForEncodedFrame(timestamp_ms);
3484 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3487 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3488
3489 video_stream_encoder_->Stop();
3490}
3491
3492TEST_F(VideoStreamEncoderTest,
3493 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3494 const int kWidth = 1280;
3495 const int kHeight = 720;
3496 const int64_t kFrameIntervalMs = 150;
3497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003498 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003499
3500 // Enable BALANCED preference, no initial limitation.
3501 AdaptingFrameForwarder source(&time_controller_);
3502 source.set_adaptation_enabled(true);
3503 video_stream_encoder_->SetSource(&source,
3504 webrtc::DegradationPreference::BALANCED);
3505
3506 int64_t timestamp_ms = kFrameIntervalMs;
3507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3508 sink_.WaitForEncodedFrame(kWidth, kHeight);
3509 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3510 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3511 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3512 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3513
3514 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3515 video_stream_encoder_->TriggerQualityLow();
3516 timestamp_ms += kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3518 sink_.WaitForEncodedFrame(timestamp_ms);
3519 EXPECT_THAT(source.sink_wants(),
3520 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3522 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3523 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3524
3525 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3526 video_stream_encoder_->TriggerQualityLow();
3527 timestamp_ms += kFrameIntervalMs;
3528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3529 sink_.WaitForEncodedFrame(timestamp_ms);
3530 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3532 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3533 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3534
3535 // Trigger adapt down, expect reduced fps (640x360@15fps).
3536 video_stream_encoder_->TriggerQualityLow();
3537 timestamp_ms += kFrameIntervalMs;
3538 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3539 WaitForEncodedFrame(timestamp_ms);
3540 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3543 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3544
3545 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3546 source.OnOutputFormatRequest(320, 180);
3547 timestamp_ms += kFrameIntervalMs;
3548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3549 WaitForEncodedFrame(320, 180);
3550 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3551 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3552
3553 // Trigger adapt down, expect reduced fps (320x180@7fps).
3554 video_stream_encoder_->TriggerCpuOveruse();
3555 timestamp_ms += kFrameIntervalMs;
3556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3557 WaitForEncodedFrame(timestamp_ms);
3558 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3561 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3562 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3563 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3565
3566 // Source requests HD, expect increased resolution (640x360@7fps).
3567 source.OnOutputFormatRequest(1280, 720);
3568 timestamp_ms += kFrameIntervalMs;
3569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3570 WaitForEncodedFrame(timestamp_ms);
3571 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3572 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3573
3574 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3575 video_stream_encoder_->TriggerCpuUnderuse();
3576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 WaitForEncodedFrame(timestamp_ms);
3579 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3583 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3584 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3585 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3586
3587 // Trigger adapt up, expect increased fps (640x360@maxfps).
3588 video_stream_encoder_->TriggerQualityHigh();
3589 video_stream_encoder_->TriggerCpuUnderuse();
3590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3592 WaitForEncodedFrame(timestamp_ms);
3593 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3596 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3597 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3598 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3599 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3600
3601 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3602 video_stream_encoder_->TriggerQualityHigh();
3603 video_stream_encoder_->TriggerCpuUnderuse();
3604 timestamp_ms += kFrameIntervalMs;
3605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3606 WaitForEncodedFrame(timestamp_ms);
3607 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3610 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3611 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3612 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3613 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3614
3615 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3616 video_stream_encoder_->TriggerQualityHigh();
3617 video_stream_encoder_->TriggerCpuUnderuse();
3618 timestamp_ms += kFrameIntervalMs;
3619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3620 WaitForEncodedFrame(timestamp_ms);
3621 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3622 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3624 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3625 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3626 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3627 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3628
3629 video_stream_encoder_->Stop();
3630}
3631
3632TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003633 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003634 const int kWidth = 1280;
3635 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003637 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003638
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003639 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003640 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003641 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003642 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003643
3644 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003645 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003646 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003647 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3648 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3649
3650 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003651 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003652 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003653 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3654 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3655
mflodmancc3d4422017-08-03 08:27:51 -07003656 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003657}
3658
mflodmancc3d4422017-08-03 08:27:51 -07003659TEST_F(VideoStreamEncoderTest,
3660 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003661 const int kWidth = 1280;
3662 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003664 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003665
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003666 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003667 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003668 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003669 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003670
3671 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003672 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003673 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003675 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3676
3677 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003678 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003680 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003681 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3682
mflodmancc3d4422017-08-03 08:27:51 -07003683 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003684}
3685
mflodmancc3d4422017-08-03 08:27:51 -07003686TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003687 const int kWidth = 1280;
3688 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003690 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003691
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003692 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003693 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003694 video_stream_encoder_->SetSource(&source,
3695 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003696
3697 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3698 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003699 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003700 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3701 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3703
3704 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003705 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003706 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003707 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3708 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3709 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3710
mflodmancc3d4422017-08-03 08:27:51 -07003711 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003712}
3713
mflodmancc3d4422017-08-03 08:27:51 -07003714TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003715 const int kWidth = 1280;
3716 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003718 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003719
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003720 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003721 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003722 video_stream_encoder_->SetSource(&source,
3723 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003724
3725 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3726 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003727 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003728 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3731
3732 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003733 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003734 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3737 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3738
mflodmancc3d4422017-08-03 08:27:51 -07003739 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003740}
3741
mflodmancc3d4422017-08-03 08:27:51 -07003742TEST_F(VideoStreamEncoderTest,
3743 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003744 const int kWidth = 1280;
3745 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003746 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003747 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003748
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003749 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003750 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003751 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003752 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003753 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003754
3755 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003756 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003758 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3759 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3760
3761 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003762 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003763 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003764 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003765 EXPECT_THAT(source.sink_wants(),
3766 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003767 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3768 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3769
3770 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003771 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003772 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3774 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3775 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3776
mflodmancc3d4422017-08-03 08:27:51 -07003777 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003778}
3779
mflodmancc3d4422017-08-03 08:27:51 -07003780TEST_F(VideoStreamEncoderTest,
3781 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003782 const int kWidth = 1280;
3783 const int kHeight = 720;
3784 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003786 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003787
3788 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3789 stats.input_frame_rate = kInputFps;
3790 stats_proxy_->SetMockStats(stats);
3791
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003792 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003793 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3794 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003795 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003796
3797 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003798 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003799 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3800 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003801 EXPECT_THAT(video_source_.sink_wants(),
3802 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003803
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003804 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003805 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003806 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003807 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003808 // Give the encoder queue time to process the change in degradation preference
3809 // by waiting for an encoded frame.
3810 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3811 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003812 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003813
3814 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003815 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003816 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3817 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003818 EXPECT_THAT(new_video_source.sink_wants(),
3819 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003820
3821 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003822 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003823 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003824
mflodmancc3d4422017-08-03 08:27:51 -07003825 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003826}
3827
mflodmancc3d4422017-08-03 08:27:51 -07003828TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003829 const int kWidth = 1280;
3830 const int kHeight = 720;
3831 const size_t kNumFrames = 10;
3832
Henrik Boström381d1092020-05-12 18:49:07 +02003833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003835
asaperssond0de2952017-04-21 01:47:31 -07003836 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003837 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003838 video_source_.set_adaptation_enabled(true);
3839
3840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3841 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3842
3843 int downscales = 0;
3844 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003845 video_source_.IncomingCapturedFrame(
3846 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3847 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003848
asaperssonfab67072017-04-04 05:51:49 -07003849 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003850 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003851 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003852 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003853
3854 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3855 ++downscales;
3856
3857 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3858 EXPECT_EQ(downscales,
3859 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3860 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003861 }
mflodmancc3d4422017-08-03 08:27:51 -07003862 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003863}
3864
mflodmancc3d4422017-08-03 08:27:51 -07003865TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003866 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3867 const int kWidth = 1280;
3868 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003870 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003871
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003872 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003873 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003874 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003875 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003876 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003877
Åsa Persson8c1bf952018-09-13 10:42:19 +02003878 int64_t timestamp_ms = kFrameIntervalMs;
3879 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003880 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003881 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3883 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3884
3885 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003886 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003887 timestamp_ms += kFrameIntervalMs;
3888 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3889 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003890 EXPECT_THAT(source.sink_wants(),
3891 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003892 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3894
3895 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003896 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003897 timestamp_ms += kFrameIntervalMs;
3898 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003899 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003900 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003901 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3902 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3903
3904 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003905 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003906 timestamp_ms += kFrameIntervalMs;
3907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3908 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003909 EXPECT_THAT(source.sink_wants(),
3910 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003911 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3912 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3913
3914 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003915 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003916 timestamp_ms += kFrameIntervalMs;
3917 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003918 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003919 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3921 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3922
mflodmancc3d4422017-08-03 08:27:51 -07003923 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003924}
3925
mflodmancc3d4422017-08-03 08:27:51 -07003926TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003927 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3928 const int kWidth = 1280;
3929 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003931 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003932
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003933 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003934 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003935 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003936 video_stream_encoder_->SetSource(&source,
3937 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003938
Åsa Persson8c1bf952018-09-13 10:42:19 +02003939 int64_t timestamp_ms = kFrameIntervalMs;
3940 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003941 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003942 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003943 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3944 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3945
3946 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003947 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003948 timestamp_ms += kFrameIntervalMs;
3949 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3950 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003951 EXPECT_THAT(source.sink_wants(),
3952 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003953 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3954 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3955
3956 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003957 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003958 timestamp_ms += kFrameIntervalMs;
3959 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003960 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003961 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003962 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3963 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3964
3965 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003966 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003967 timestamp_ms += kFrameIntervalMs;
3968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3969 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003970 EXPECT_THAT(source.sink_wants(),
3971 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003972 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3973 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3974
3975 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003976 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003977 timestamp_ms += kFrameIntervalMs;
3978 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003979 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003980 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003981 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3982 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3983
mflodmancc3d4422017-08-03 08:27:51 -07003984 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003985}
3986
Sergey Silkin41c650b2019-10-14 13:12:19 +02003987TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3988 fake_encoder_.SetResolutionBitrateLimits(
3989 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3990
Henrik Boström381d1092020-05-12 18:49:07 +02003991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003992 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3993 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3994 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3995 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003996
3997 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003998 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003999 source.set_adaptation_enabled(true);
4000 video_stream_encoder_->SetSource(
4001 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4002
4003 // Insert 720p frame.
4004 int64_t timestamp_ms = kFrameIntervalMs;
4005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4006 WaitForEncodedFrame(1280, 720);
4007
4008 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004010 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4011 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4012 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4013 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004014 video_stream_encoder_->TriggerQualityLow();
4015
4016 // Insert 720p frame. It should be downscaled and encoded.
4017 timestamp_ms += kFrameIntervalMs;
4018 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4019 WaitForEncodedFrame(960, 540);
4020
4021 // Trigger adapt up. Higher resolution should not be requested duo to lack
4022 // of bitrate.
4023 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004024 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004025
4026 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004028 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4029 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4030 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4031 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004032
4033 // Trigger adapt up. Higher resolution should be requested.
4034 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004035 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004036
4037 video_stream_encoder_->Stop();
4038}
4039
4040TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4041 fake_encoder_.SetResolutionBitrateLimits(
4042 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4043
4044 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004046 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4047 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4048 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4049 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004050
4051 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004052 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004053 source.set_adaptation_enabled(true);
4054 video_stream_encoder_->SetSource(
4055 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4056
4057 // Insert 720p frame. It should be dropped and lower resolution should be
4058 // requested.
4059 int64_t timestamp_ms = kFrameIntervalMs;
4060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4061 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004062 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004063
4064 // Insert 720p frame. It should be downscaled and encoded.
4065 timestamp_ms += kFrameIntervalMs;
4066 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4067 WaitForEncodedFrame(960, 540);
4068
4069 video_stream_encoder_->Stop();
4070}
4071
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004072class BalancedDegradationTest : public VideoStreamEncoderTest {
4073 protected:
4074 void SetupTest() {
4075 // Reset encoder for field trials to take effect.
4076 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004077 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004078
4079 // Enable BALANCED preference.
4080 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004081 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4082 }
4083
Asa Persson606d3cb2021-10-04 10:07:11 +02004084 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004086 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004087 }
4088
Åsa Persson45b176f2019-09-30 11:19:05 +02004089 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004090 timestamp_ms_ += kFrameIntervalMs;
4091 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004092 }
4093
4094 void InsertFrameAndWaitForEncoded() {
4095 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004096 sink_.WaitForEncodedFrame(timestamp_ms_);
4097 }
4098
4099 const int kWidth = 640; // pixels:640x360=230400
4100 const int kHeight = 360;
4101 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4102 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004103 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004104};
4105
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004106TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004107 test::ScopedFieldTrials field_trials(
4108 "WebRTC-Video-BalancedDegradationSettings/"
4109 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4110 SetupTest();
4111
4112 // Force input frame rate.
4113 const int kInputFps = 24;
4114 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4115 stats.input_frame_rate = kInputFps;
4116 stats_proxy_->SetMockStats(stats);
4117
Åsa Persson45b176f2019-09-30 11:19:05 +02004118 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004119 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004120
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004121 // Trigger adapt down, expect scaled down framerate and resolution,
4122 // since Fps diff (input-requested:0) < threshold.
4123 video_stream_encoder_->TriggerQualityLow();
4124 EXPECT_THAT(source_.sink_wants(),
4125 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004126
4127 video_stream_encoder_->Stop();
4128}
4129
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004130TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004131 test::ScopedFieldTrials field_trials(
4132 "WebRTC-Video-BalancedDegradationSettings/"
4133 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4134 SetupTest();
4135
4136 // Force input frame rate.
4137 const int kInputFps = 25;
4138 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4139 stats.input_frame_rate = kInputFps;
4140 stats_proxy_->SetMockStats(stats);
4141
Åsa Persson45b176f2019-09-30 11:19:05 +02004142 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004143 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004144
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004145 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4146 // Fps diff (input-requested:1) == threshold.
4147 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004148 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004149
4150 video_stream_encoder_->Stop();
4151}
4152
4153TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4154 test::ScopedFieldTrials field_trials(
4155 "WebRTC-Video-BalancedDegradationSettings/"
4156 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4157 SetupTest();
4158
4159 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4160
Åsa Persson45b176f2019-09-30 11:19:05 +02004161 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004162 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004163
4164 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4165 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004166 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004167
4168 video_stream_encoder_->Stop();
4169}
4170
Åsa Perssonccfb3402019-09-25 15:13:04 +02004171TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004172 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004173 "WebRTC-Video-BalancedDegradationSettings/"
4174 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004175 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004176
Asa Persson606d3cb2021-10-04 10:07:11 +02004177 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4178 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4179 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004180
Åsa Persson45b176f2019-09-30 11:19:05 +02004181 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004182 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004183 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4184
4185 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4186 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004187 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004188 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004189 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4190
4191 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4192 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004193 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004194 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004195 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4196
Åsa Persson30ab0152019-08-27 12:22:33 +02004197 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4198 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004199 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004200 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004201 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004202 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4203
4204 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004205 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004206 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004207 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004208
Åsa Persson30ab0152019-08-27 12:22:33 +02004209 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004210 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004211 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004212 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004213 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004214 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4215
4216 video_stream_encoder_->Stop();
4217}
4218
Åsa Perssonccfb3402019-09-25 15:13:04 +02004219TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004220 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4221 test::ScopedFieldTrials field_trials(
4222 "WebRTC-Video-BalancedDegradationSettings/"
4223 "pixels:57600|129600|230400,fps:7|24|24/");
4224 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004225 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004226
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004227 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004228
4229 // Insert frame, expect scaled down:
4230 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4231 InsertFrame();
4232 EXPECT_FALSE(WaitForFrame(1000));
4233 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4234 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4235
4236 // Insert frame, expect scaled down:
4237 // resolution (320x180@24fps).
4238 InsertFrame();
4239 EXPECT_FALSE(WaitForFrame(1000));
4240 EXPECT_LT(source_.sink_wants().max_pixel_count,
4241 source_.last_wants().max_pixel_count);
4242 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4243
4244 // Frame should not be dropped (min pixels per frame reached).
4245 InsertFrameAndWaitForEncoded();
4246
4247 video_stream_encoder_->Stop();
4248}
4249
4250TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004251 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004252 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004253 "WebRTC-Video-BalancedDegradationSettings/"
4254 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004255 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004256
Asa Persson606d3cb2021-10-04 10:07:11 +02004257 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4258 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4259 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004260
Åsa Persson45b176f2019-09-30 11:19:05 +02004261 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004262 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004263 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4264
4265 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4266 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004267 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004268 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004269 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4270
4271 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4272 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004273 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004274 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004275 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4276
4277 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4278 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004279 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004280 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004281 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4282
Åsa Persson30ab0152019-08-27 12:22:33 +02004283 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4284 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004285 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004286 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004287 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4288
4289 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4290 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004291 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004292 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4293
4294 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004295 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004296 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004297 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004298 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004299 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4300
4301 video_stream_encoder_->Stop();
4302}
4303
Åsa Perssonccfb3402019-09-25 15:13:04 +02004304TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004305 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004306 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 "WebRTC-Video-BalancedDegradationSettings/"
4308 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004309 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004310
Asa Persson606d3cb2021-10-04 10:07:11 +02004311 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4312 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4313 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4314 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4315 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004316
Åsa Persson45b176f2019-09-30 11:19:05 +02004317 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004318 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004319 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4320
4321 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4322 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004323 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004324 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004325 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4326
4327 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4328 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004329 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004330 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004331 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4332
4333 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4334 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004335 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004336 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004337 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4338
4339 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4340 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004341 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004342 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
4344 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004345 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004347 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004348 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004349 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
4351 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004352 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004353 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004354 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
4357 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004358 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004359 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004360 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004361 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004362 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4363
Åsa Persson1b247f12019-08-14 17:26:39 +02004364 video_stream_encoder_->Stop();
4365}
4366
mflodmancc3d4422017-08-03 08:27:51 -07004367TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004368 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4369 const int kWidth = 1280;
4370 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004371 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004372 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004373
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004374 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004375 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004376 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004377 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004378 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004379
Åsa Persson8c1bf952018-09-13 10:42:19 +02004380 int64_t timestamp_ms = kFrameIntervalMs;
4381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004382 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004383 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004384 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4385 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4386 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4387 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4388
4389 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004390 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004391 timestamp_ms += kFrameIntervalMs;
4392 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4393 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004394 EXPECT_THAT(source.sink_wants(),
4395 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004396 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4397 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4398 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4399 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4400
4401 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004402 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004403 timestamp_ms += kFrameIntervalMs;
4404 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4405 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004406 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004407 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4408 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4409 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4410 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4411
Jonathan Yubc771b72017-12-08 17:04:29 -08004412 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004413 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004414 timestamp_ms += kFrameIntervalMs;
4415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4416 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004417 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004418 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4419 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004420 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004421 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4422
Jonathan Yubc771b72017-12-08 17:04:29 -08004423 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004424 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004425 timestamp_ms += kFrameIntervalMs;
4426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4427 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004428 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004429 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004430 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4431 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4432 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4433 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4434
Jonathan Yubc771b72017-12-08 17:04:29 -08004435 // Trigger quality adapt down, expect no change (min resolution reached).
4436 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004437 timestamp_ms += kFrameIntervalMs;
4438 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4439 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004440 EXPECT_THAT(source.sink_wants(), FpsMax());
4441 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004442 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4443 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4444 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4445 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4446
Evan Shrubsole64469032020-06-11 10:45:29 +02004447 // Trigger quality adapt up, expect upscaled resolution (480x270).
4448 video_stream_encoder_->TriggerQualityHigh();
4449 timestamp_ms += kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4451 WaitForEncodedFrame(timestamp_ms);
4452 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4453 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4455 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4456 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4457
4458 // Trigger quality and cpu adapt up since both are most limited, expect
4459 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004460 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004461 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004462 timestamp_ms += kFrameIntervalMs;
4463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4464 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004465 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004466 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4467 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4468 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004469 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004470
Evan Shrubsole64469032020-06-11 10:45:29 +02004471 // Trigger quality and cpu adapt up since both are most limited, expect
4472 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004473 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004474 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004475 timestamp_ms += kFrameIntervalMs;
4476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4477 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004479 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004482 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4483 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004484
Evan Shrubsole64469032020-06-11 10:45:29 +02004485 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4486 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004487 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004488 timestamp_ms += kFrameIntervalMs;
4489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4490 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004491 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004492 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004494 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004495 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004496
4497 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004498 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004499 timestamp_ms += kFrameIntervalMs;
4500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004501 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004502 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004503 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004504 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4505 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004506 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004507 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004508
mflodmancc3d4422017-08-03 08:27:51 -07004509 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004510}
4511
mflodmancc3d4422017-08-03 08:27:51 -07004512TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004513 const int kWidth = 640;
4514 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004515
Henrik Boström381d1092020-05-12 18:49:07 +02004516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004517 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004518
perkj803d97f2016-11-01 11:45:46 -07004519 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004520 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004521 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004522 }
4523
mflodmancc3d4422017-08-03 08:27:51 -07004524 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004525 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004526 video_source_.IncomingCapturedFrame(CreateFrame(
4527 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004528 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004529 }
4530
mflodmancc3d4422017-08-03 08:27:51 -07004531 video_stream_encoder_->Stop();
4532 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004533 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004534
Ying Wangef3998f2019-12-09 13:06:53 +01004535 EXPECT_METRIC_EQ(
4536 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4537 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004538 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4539}
4540
mflodmancc3d4422017-08-03 08:27:51 -07004541TEST_F(VideoStreamEncoderTest,
4542 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004544 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004545 const int kWidth = 640;
4546 const int kHeight = 360;
4547
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004548 video_stream_encoder_->SetSource(&video_source_,
4549 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004550
4551 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4552 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004553 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004554 }
4555
mflodmancc3d4422017-08-03 08:27:51 -07004556 video_stream_encoder_->Stop();
4557 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004558 stats_proxy_.reset();
4559
4560 EXPECT_EQ(0,
4561 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4562}
4563
Per Kjellanderdcef6412020-10-07 15:09:05 +02004564TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4565 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004566 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004567 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004568
4569 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004570 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004571 SimulcastRateAllocator(fake_encoder_.config())
4572 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004573 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004574
Henrik Boström381d1092020-05-12 18:49:07 +02004575 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004576 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004577
sprang57c2fff2017-01-16 06:24:02 -08004578 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004579 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4580 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004581 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4582 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4583
Erik Språngd7329ca2019-02-21 21:19:53 +01004584 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004585 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004586 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004587
Per Kjellanderdcef6412020-10-07 15:09:05 +02004588 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004589 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004590 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4591 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004592 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004593 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004594
Per Kjellanderdcef6412020-10-07 15:09:05 +02004595 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004596 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004597 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004598 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004599 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4600 WaitForEncodedFrame(CurrentTimeMs());
4601 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004602 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004603 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004604
mflodmancc3d4422017-08-03 08:27:51 -07004605 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004606}
4607
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004608TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004609 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004610 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004611 kVideoLayersAllocation);
4612
4613 const int kDefaultFps = 30;
4614
4615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004616 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004617
4618 video_source_.IncomingCapturedFrame(
4619 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4620 WaitForEncodedFrame(CurrentTimeMs());
4621 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4622 VideoLayersAllocation last_layer_allocation =
4623 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004624 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004625 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4626
4627 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004628 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004629 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004630 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004631 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4632
Erik Språng9d69cbe2020-10-22 17:44:42 +02004633 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004634 int number_of_layers_allocation = 1;
4635 const int64_t start_time_ms = CurrentTimeMs();
4636 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4637 video_source_.IncomingCapturedFrame(
4638 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4639 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004640 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4641 number_of_layers_allocation = sink_.number_of_layers_allocations();
4642 VideoLayersAllocation new_allocation =
4643 sink_.GetLastVideoLayersAllocation();
4644 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4645 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4646 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4647 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4648 .target_bitrate_per_temporal_layer,
4649 last_layer_allocation.active_spatial_layers[0]
4650 .target_bitrate_per_temporal_layer);
4651 last_layer_allocation = new_allocation;
4652 }
4653 }
4654 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4655 video_stream_encoder_->Stop();
4656}
4657
4658TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004659 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004660 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4661 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4662 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004663 VideoEncoderConfig video_encoder_config;
4664 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4665 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004666 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004667 video_encoder_config.content_type =
4668 VideoEncoderConfig::ContentType::kRealtimeVideo;
4669 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004670 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004671 VideoEncoder::GetDefaultVp8Settings());
4672 for (auto& layer : video_encoder_config.simulcast_layers) {
4673 layer.num_temporal_layers = 2;
4674 }
4675 // Simulcast layers are used for enabling/disabling streams.
4676 video_encoder_config.simulcast_layers[0].active = true;
4677 video_encoder_config.simulcast_layers[1].active = false;
4678 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004679 ConfigureEncoder(std::move(video_encoder_config),
4680 VideoStreamEncoder::BitrateAllocationCallbackType::
4681 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004682
4683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004685
4686 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4687 WaitForEncodedFrame(CurrentTimeMs());
4688 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4689 VideoLayersAllocation last_layer_allocation =
4690 sink_.GetLastVideoLayersAllocation();
4691
4692 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4693 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4694 .target_bitrate_per_temporal_layer,
4695 SizeIs(2));
4696 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4697 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4698 video_stream_encoder_->Stop();
4699}
4700
4701TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004702 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004703 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4704 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4705 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004706 VideoEncoderConfig video_encoder_config;
4707 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4708 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004709 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004710 video_encoder_config.content_type =
4711 VideoEncoderConfig::ContentType::kRealtimeVideo;
4712 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004713 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004714 VideoEncoder::GetDefaultVp8Settings());
4715 for (auto& layer : video_encoder_config.simulcast_layers) {
4716 layer.num_temporal_layers = 2;
4717 }
4718 // Simulcast layers are used for enabling/disabling streams.
4719 video_encoder_config.simulcast_layers[0].active = true;
4720 video_encoder_config.simulcast_layers[1].active = false;
4721 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004722 ConfigureEncoder(std::move(video_encoder_config),
4723 VideoStreamEncoder::BitrateAllocationCallbackType::
4724 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725
4726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004727 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004728
4729 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4730 WaitForEncodedFrame(CurrentTimeMs());
4731 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4732 VideoLayersAllocation last_layer_allocation =
4733 sink_.GetLastVideoLayersAllocation();
4734
4735 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4736 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4737 .target_bitrate_per_temporal_layer,
4738 SizeIs(2));
4739 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4740
4741 video_stream_encoder_->Stop();
4742}
4743
4744TEST_F(VideoStreamEncoderTest,
4745 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4746 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4747 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004748 VideoEncoderConfig video_encoder_config;
4749 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4750 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004751 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004752 video_encoder_config.content_type =
4753 VideoEncoderConfig::ContentType::kRealtimeVideo;
4754 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4755 vp9_settings.numberOfSpatialLayers = 2;
4756 vp9_settings.numberOfTemporalLayers = 2;
4757 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4758 vp9_settings.automaticResizeOn = false;
4759 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004760 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004761 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004762 ConfigureEncoder(std::move(video_encoder_config),
4763 VideoStreamEncoder::BitrateAllocationCallbackType::
4764 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004765
4766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004768
4769 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4770 WaitForEncodedFrame(CurrentTimeMs());
4771 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4772 VideoLayersAllocation last_layer_allocation =
4773 sink_.GetLastVideoLayersAllocation();
4774
4775 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4776 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4777 .target_bitrate_per_temporal_layer,
4778 SizeIs(2));
4779 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4780 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4781 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4782 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4783 .target_bitrate_per_temporal_layer,
4784 SizeIs(2));
4785 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4786 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4787 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4788
4789 // Since full SVC is used, expect the top layer to utilize the full target
4790 // rate.
4791 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4792 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004793 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004794 video_stream_encoder_->Stop();
4795}
4796
4797TEST_F(VideoStreamEncoderTest,
4798 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4799 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4800 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004801 VideoEncoderConfig video_encoder_config;
4802 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4803 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004804 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004805 video_encoder_config.content_type =
4806 VideoEncoderConfig::ContentType::kRealtimeVideo;
4807 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4808 vp9_settings.numberOfSpatialLayers = 2;
4809 vp9_settings.numberOfTemporalLayers = 2;
4810 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4811 vp9_settings.automaticResizeOn = false;
4812 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004813 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004814 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004815 ConfigureEncoder(std::move(video_encoder_config),
4816 VideoStreamEncoder::BitrateAllocationCallbackType::
4817 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004818
4819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004820 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821
4822 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4823 WaitForEncodedFrame(CurrentTimeMs());
4824 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4825 VideoLayersAllocation last_layer_allocation =
4826 sink_.GetLastVideoLayersAllocation();
4827
4828 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4829 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4830 .target_bitrate_per_temporal_layer,
4831 SizeIs(1));
4832 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4833 .target_bitrate_per_temporal_layer,
4834 SizeIs(1));
4835 // Since full SVC is used, expect the top layer to utilize the full target
4836 // rate.
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4838 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004839 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004840 video_stream_encoder_->Stop();
4841}
4842
4843TEST_F(VideoStreamEncoderTest,
4844 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4845 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4846 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004847 VideoEncoderConfig video_encoder_config;
4848 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4849 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004850 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851 video_encoder_config.content_type =
4852 VideoEncoderConfig::ContentType::kRealtimeVideo;
4853 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4854 vp9_settings.numberOfSpatialLayers = 2;
4855 vp9_settings.numberOfTemporalLayers = 2;
4856 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4857 vp9_settings.automaticResizeOn = false;
4858 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004859 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004860 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004861 ConfigureEncoder(std::move(video_encoder_config),
4862 VideoStreamEncoder::BitrateAllocationCallbackType::
4863 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004864
4865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004866 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004867
4868 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4869 WaitForEncodedFrame(CurrentTimeMs());
4870 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4871 VideoLayersAllocation last_layer_allocation =
4872 sink_.GetLastVideoLayersAllocation();
4873
4874 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4875 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4876 .target_bitrate_per_temporal_layer,
4877 SizeIs(2));
4878 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4879 .target_bitrate_per_temporal_layer,
4880 SizeIs(2));
4881 // Since KSVC is, spatial layers are independend except on key frames.
4882 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4883 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004884 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004885 video_stream_encoder_->Stop();
4886}
4887
4888TEST_F(VideoStreamEncoderTest,
4889 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4890 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4891 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4892 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004893 VideoEncoderConfig video_encoder_config;
4894 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4895 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004896 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004897 video_encoder_config.content_type =
4898 VideoEncoderConfig::ContentType::kRealtimeVideo;
4899 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4900 vp9_settings.numberOfSpatialLayers = 3;
4901 vp9_settings.numberOfTemporalLayers = 2;
4902 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4903 vp9_settings.automaticResizeOn = false;
4904 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004905 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004906 vp9_settings);
4907 // Simulcast layers are used for enabling/disabling streams.
4908 video_encoder_config.simulcast_layers.resize(3);
4909 video_encoder_config.simulcast_layers[0].active = false;
4910 video_encoder_config.simulcast_layers[1].active = true;
4911 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004912 ConfigureEncoder(std::move(video_encoder_config),
4913 VideoStreamEncoder::BitrateAllocationCallbackType::
4914 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004915
4916 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004917 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004918
4919 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4920 WaitForEncodedFrame(CurrentTimeMs());
4921 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4922 VideoLayersAllocation last_layer_allocation =
4923 sink_.GetLastVideoLayersAllocation();
4924
4925 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4926 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4927 .target_bitrate_per_temporal_layer,
4928 SizeIs(2));
4929 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4930 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4931
4932 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4933 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4934 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4935 .target_bitrate_per_temporal_layer,
4936 SizeIs(2));
4937 // Since full SVC is used, expect the top layer to utilize the full target
4938 // rate.
4939 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4940 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004941 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004942 video_stream_encoder_->Stop();
4943}
4944
4945TEST_F(VideoStreamEncoderTest,
4946 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4947 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4948 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4949 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004950 VideoEncoderConfig video_encoder_config;
4951 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4952 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004953 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004954 video_encoder_config.content_type =
4955 VideoEncoderConfig::ContentType::kRealtimeVideo;
4956 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4957 vp9_settings.numberOfSpatialLayers = 3;
4958 vp9_settings.numberOfTemporalLayers = 2;
4959 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4960 vp9_settings.automaticResizeOn = false;
4961 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004962 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004963 vp9_settings);
4964 // Simulcast layers are used for enabling/disabling streams.
4965 video_encoder_config.simulcast_layers.resize(3);
4966 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004967 ConfigureEncoder(std::move(video_encoder_config),
4968 VideoStreamEncoder::BitrateAllocationCallbackType::
4969 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004970
4971 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004972 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004973
4974 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4975 WaitForEncodedFrame(CurrentTimeMs());
4976 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4977 VideoLayersAllocation last_layer_allocation =
4978 sink_.GetLastVideoLayersAllocation();
4979
4980 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4981 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4982 .target_bitrate_per_temporal_layer,
4983 SizeIs(2));
4984 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4985 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4986
4987 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4988 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4989 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4990 .target_bitrate_per_temporal_layer,
4991 SizeIs(2));
4992 video_stream_encoder_->Stop();
4993}
4994
4995TEST_F(VideoStreamEncoderTest,
4996 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4997 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4998 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4999 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000 VideoEncoderConfig video_encoder_config;
5001 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5002 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005003 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005004 video_encoder_config.content_type =
5005 VideoEncoderConfig::ContentType::kRealtimeVideo;
5006 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5007 vp9_settings.numberOfSpatialLayers = 3;
5008 vp9_settings.numberOfTemporalLayers = 2;
5009 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5010 vp9_settings.automaticResizeOn = false;
5011 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005012 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005013 vp9_settings);
5014 // Simulcast layers are used for enabling/disabling streams.
5015 video_encoder_config.simulcast_layers.resize(3);
5016 video_encoder_config.simulcast_layers[0].active = false;
5017 video_encoder_config.simulcast_layers[1].active = false;
5018 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005019 ConfigureEncoder(std::move(video_encoder_config),
5020 VideoStreamEncoder::BitrateAllocationCallbackType::
5021 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005022
5023 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005024 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005025
5026 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5027 WaitForEncodedFrame(CurrentTimeMs());
5028 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5029 VideoLayersAllocation last_layer_allocation =
5030 sink_.GetLastVideoLayersAllocation();
5031
5032 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5033 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5034 .target_bitrate_per_temporal_layer,
5035 SizeIs(2));
5036 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5037 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5038 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5039 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005040 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005041 video_stream_encoder_->Stop();
5042}
5043
5044TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5045 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005046 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005047 kVideoLayersAllocation);
5048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005049 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005050
5051 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5052 WaitForEncodedFrame(CurrentTimeMs());
5053 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5054 VideoLayersAllocation last_layer_allocation =
5055 sink_.GetLastVideoLayersAllocation();
5056
5057 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5058 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5059 .target_bitrate_per_temporal_layer,
5060 SizeIs(1));
5061 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5062 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005063 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005064 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5065 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5066 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5067 video_stream_encoder_->Stop();
5068}
5069
5070TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005071 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5072 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005073 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005074 kVideoLayersAllocation);
5075
5076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005077 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005078
5079 video_source_.IncomingCapturedFrame(
5080 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5081 WaitForEncodedFrame(CurrentTimeMs());
5082 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5083 VideoLayersAllocation last_layer_allocation =
5084 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005085 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005086 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5087 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5088 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005089 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005090
5091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005092 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5093 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005094 video_source_.IncomingCapturedFrame(
5095 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5096 WaitForEncodedFrame(CurrentTimeMs());
5097
5098 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5099 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5100 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5101 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5102 .target_bitrate_per_temporal_layer[0],
5103 DataRate::Zero());
5104
5105 video_stream_encoder_->Stop();
5106}
5107
Per Kjellander4190ce92020-12-15 17:24:55 +01005108TEST_F(VideoStreamEncoderTest,
5109 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5110 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005111 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005112 kVideoLayersAllocation);
5113
5114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005115 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5116 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005117
5118 video_source_.IncomingCapturedFrame(
5119 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5120 WaitForEncodedFrame(CurrentTimeMs());
5121 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5122 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5123 SizeIs(2));
5124 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5125 codec_width_);
5126 EXPECT_EQ(
5127 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5128 codec_height_);
5129
5130 video_source_.IncomingCapturedFrame(
5131 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5132 WaitForEncodedFrame(CurrentTimeMs());
5133 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5134 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5135 SizeIs(2));
5136 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5137 codec_width_ / 2);
5138 EXPECT_EQ(
5139 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5140 codec_height_ / 2);
5141
5142 video_stream_encoder_->Stop();
5143}
5144
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005145TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5146 // 2 TLs configured, temporal layers supported by encoder.
5147 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005148 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005149 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005150 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005151 fake_encoder_.SetTemporalLayersSupported(0, true);
5152
5153 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005154 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005155 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005156 kNumTemporalLayers, /*temporal_id*/ 0,
5157 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005158 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005159 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005160 kNumTemporalLayers, /*temporal_id*/ 1,
5161 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005162 VideoBitrateAllocation expected_bitrate;
5163 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5164 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5165
5166 VerifyAllocatedBitrate(expected_bitrate);
5167 video_stream_encoder_->Stop();
5168}
5169
5170TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5171 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005172 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005173 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005174 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005175 fake_encoder_.SetTemporalLayersSupported(0, false);
5176
5177 // Temporal layers not supported by the encoder.
5178 // Total bitrate should be at ti:0.
5179 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005180 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005181
5182 VerifyAllocatedBitrate(expected_bitrate);
5183 video_stream_encoder_->Stop();
5184}
5185
5186TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005187 webrtc::test::ScopedFieldTrials field_trials(
5188 "WebRTC-Video-QualityScalerSettings/"
5189 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5190 // Reset encoder for field trials to take effect.
5191 ConfigureEncoder(video_encoder_config_.Copy());
5192
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005193 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005194 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005195 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005196 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005197 fake_encoder_.SetTemporalLayersSupported(0, true);
5198 fake_encoder_.SetTemporalLayersSupported(1, false);
5199
5200 const int kS0Bps = 150000;
5201 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005202 kS0Bps *
5203 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5204 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005205 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005206 kS0Bps *
5207 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5208 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005209 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005210 // Temporal layers not supported by si:1.
5211 VideoBitrateAllocation expected_bitrate;
5212 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5213 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5214 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5215
5216 VerifyAllocatedBitrate(expected_bitrate);
5217 video_stream_encoder_->Stop();
5218}
5219
Niels Möller7dc26b72017-12-06 10:27:48 +01005220TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5221 const int kFrameWidth = 1280;
5222 const int kFrameHeight = 720;
5223 const int kFramerate = 24;
5224
Henrik Boström381d1092020-05-12 18:49:07 +02005225 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005226 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005227 test::FrameForwarder source;
5228 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005229 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005230
5231 // Insert a single frame, triggering initial configuration.
5232 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5233 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5234
5235 EXPECT_EQ(
5236 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5237 kDefaultFramerate);
5238
5239 // Trigger reconfigure encoder (without resetting the entire instance).
5240 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005241 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5242 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005243 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005244 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005245 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005246 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5247
5248 // Detector should be updated with fps limit from codec config.
5249 EXPECT_EQ(
5250 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5251 kFramerate);
5252
5253 // Trigger overuse, max framerate should be reduced.
5254 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5255 stats.input_frame_rate = kFramerate;
5256 stats_proxy_->SetMockStats(stats);
5257 video_stream_encoder_->TriggerCpuOveruse();
5258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5259 int adapted_framerate =
5260 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5261 EXPECT_LT(adapted_framerate, kFramerate);
5262
5263 // Trigger underuse, max framerate should go back to codec configured fps.
5264 // Set extra low fps, to make sure it's actually reset, not just incremented.
5265 stats = stats_proxy_->GetStats();
5266 stats.input_frame_rate = adapted_framerate / 2;
5267 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005268 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005269 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5270 EXPECT_EQ(
5271 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5272 kFramerate);
5273
5274 video_stream_encoder_->Stop();
5275}
5276
5277TEST_F(VideoStreamEncoderTest,
5278 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5279 const int kFrameWidth = 1280;
5280 const int kFrameHeight = 720;
5281 const int kLowFramerate = 15;
5282 const int kHighFramerate = 25;
5283
Henrik Boström381d1092020-05-12 18:49:07 +02005284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005285 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005286 test::FrameForwarder source;
5287 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005288 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005289
5290 // Trigger initial configuration.
5291 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005292 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5293 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005294 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005295 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005296 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005297 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005298 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5299
5300 EXPECT_EQ(
5301 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5302 kLowFramerate);
5303
5304 // Trigger overuse, max framerate should be reduced.
5305 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5306 stats.input_frame_rate = kLowFramerate;
5307 stats_proxy_->SetMockStats(stats);
5308 video_stream_encoder_->TriggerCpuOveruse();
5309 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5310 int adapted_framerate =
5311 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5312 EXPECT_LT(adapted_framerate, kLowFramerate);
5313
5314 // Reconfigure the encoder with a new (higher max framerate), max fps should
5315 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005316 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005317 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5318 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005319 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005320 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5321
5322 EXPECT_EQ(
5323 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5324 adapted_framerate);
5325
5326 // Trigger underuse, max framerate should go back to codec configured fps.
5327 stats = stats_proxy_->GetStats();
5328 stats.input_frame_rate = adapted_framerate;
5329 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005330 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005331 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5332 EXPECT_EQ(
5333 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5334 kHighFramerate);
5335
5336 video_stream_encoder_->Stop();
5337}
5338
mflodmancc3d4422017-08-03 08:27:51 -07005339TEST_F(VideoStreamEncoderTest,
5340 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005341 const int kFrameWidth = 1280;
5342 const int kFrameHeight = 720;
5343 const int kFramerate = 24;
5344
Henrik Boström381d1092020-05-12 18:49:07 +02005345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005346 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005347 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005348 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005349 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005350
5351 // Trigger initial configuration.
5352 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005353 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5354 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005355 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005356 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005357 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005358 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005360
Niels Möller7dc26b72017-12-06 10:27:48 +01005361 EXPECT_EQ(
5362 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5363 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005364
5365 // Trigger overuse, max framerate should be reduced.
5366 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5367 stats.input_frame_rate = kFramerate;
5368 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005369 video_stream_encoder_->TriggerCpuOveruse();
5370 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005371 int adapted_framerate =
5372 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005373 EXPECT_LT(adapted_framerate, kFramerate);
5374
5375 // Change degradation preference to not enable framerate scaling. Target
5376 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005377 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005378 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005379 EXPECT_EQ(
5380 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5381 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005382
mflodmancc3d4422017-08-03 08:27:51 -07005383 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005384}
5385
mflodmancc3d4422017-08-03 08:27:51 -07005386TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005387 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005389 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5390 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5391 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005392 const int kWidth = 640;
5393 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005394
asaperssonfab67072017-04-04 05:51:49 -07005395 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005396
5397 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005398 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005399
5400 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005401 EXPECT_TRUE_WAIT(
5402 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005403
sprangc5d62e22017-04-02 23:53:04 -07005404 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005405
asaperssonfab67072017-04-04 05:51:49 -07005406 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005407 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005408 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005409
5410 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005411 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005412
Henrik Boström2671dac2020-05-19 16:29:09 +02005413 EXPECT_TRUE_WAIT(
5414 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005415
mflodmancc3d4422017-08-03 08:27:51 -07005416 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005417}
5418
mflodmancc3d4422017-08-03 08:27:51 -07005419TEST_F(VideoStreamEncoderTest,
5420 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005421 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005423 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5424 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5425 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005426 const int kWidth = 640;
5427 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005428
5429 // We expect the n initial frames to get dropped.
5430 int i;
5431 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005432 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005433 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005434 }
5435 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005436 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005437 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005438
5439 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005440 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005441
mflodmancc3d4422017-08-03 08:27:51 -07005442 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005443}
5444
mflodmancc3d4422017-08-03 08:27:51 -07005445TEST_F(VideoStreamEncoderTest,
5446 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005447 const int kWidth = 640;
5448 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005450 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005451
5452 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005453 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005454 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005455
asaperssonfab67072017-04-04 05:51:49 -07005456 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005457 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005458 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005459
mflodmancc3d4422017-08-03 08:27:51 -07005460 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005461}
5462
mflodmancc3d4422017-08-03 08:27:51 -07005463TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005464 const int kWidth = 640;
5465 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005466 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005467
5468 VideoEncoderConfig video_encoder_config;
5469 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5470 // Make format different, to force recreation of encoder.
5471 video_encoder_config.video_format.parameters["foo"] = "foo";
5472 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005473 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005475 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005476
kthelgasonb83797b2017-02-14 11:57:25 -08005477 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005478 video_stream_encoder_->SetSource(&video_source_,
5479 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005480
asaperssonfab67072017-04-04 05:51:49 -07005481 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005482 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005483 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005484
mflodmancc3d4422017-08-03 08:27:51 -07005485 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005486 fake_encoder_.SetQualityScaling(true);
5487}
5488
Åsa Persson139f4dc2019-08-02 09:29:58 +02005489TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5490 webrtc::test::ScopedFieldTrials field_trials(
5491 "WebRTC-Video-QualityScalerSettings/"
5492 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5493 // Reset encoder for field trials to take effect.
5494 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005495 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5496 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005497 const int kWidth = 640;
5498 const int kHeight = 360;
5499
Henrik Boström381d1092020-05-12 18:49:07 +02005500 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005501 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005502 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5503 // Frame should not be dropped.
5504 WaitForEncodedFrame(1);
5505
Henrik Boström381d1092020-05-12 18:49:07 +02005506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005507 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5508 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5509 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005510 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5511 // Frame should not be dropped.
5512 WaitForEncodedFrame(2);
5513
Henrik Boström381d1092020-05-12 18:49:07 +02005514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005515 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5516 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5517 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005518 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5519 // Expect to drop this frame, the wait should time out.
5520 ExpectDroppedFrame();
5521
5522 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005523 EXPECT_TRUE_WAIT(
5524 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005525 video_stream_encoder_->Stop();
5526}
5527
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005528TEST_F(VideoStreamEncoderTest,
5529 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5530 webrtc::test::ScopedFieldTrials field_trials(
5531 "WebRTC-Video-QualityScalerSettings/"
5532 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5533 fake_encoder_.SetQualityScaling(false);
5534 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005535 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5536 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005537 const int kWidth = 640;
5538 const int kHeight = 360;
5539
5540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005541 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005542 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5543 // Frame should not be dropped.
5544 WaitForEncodedFrame(1);
5545
5546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5547 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5548 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5549 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5550 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5551 // Frame should not be dropped.
5552 WaitForEncodedFrame(2);
5553
5554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5555 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5556 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5557 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5558 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5559 // Not dropped since quality scaling is disabled.
5560 WaitForEncodedFrame(3);
5561
5562 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005563 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005564 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5565
5566 video_stream_encoder_->Stop();
5567}
5568
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005569TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005570 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005571 // Set simulcast.
5572 ResetEncoder("VP8", 3, 1, 1, false);
5573 fake_encoder_.SetQualityScaling(true);
5574 const int kWidth = 1280;
5575 const int kHeight = 720;
5576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005577 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005578 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5579 // Frame should not be dropped.
5580 WaitForEncodedFrame(1);
5581
5582 // Trigger QVGA "singlecast"
5583 // Update the config.
5584 VideoEncoderConfig video_encoder_config;
5585 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5586 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005587 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005588 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005589 "VP8", /*max qp*/ 56, /*screencast*/ false,
5590 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005591 for (auto& layer : video_encoder_config.simulcast_layers) {
5592 layer.num_temporal_layers = 1;
5593 layer.max_framerate = kDefaultFramerate;
5594 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005595 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005596 video_encoder_config.content_type =
5597 VideoEncoderConfig::ContentType::kRealtimeVideo;
5598
5599 video_encoder_config.simulcast_layers[0].active = true;
5600 video_encoder_config.simulcast_layers[1].active = false;
5601 video_encoder_config.simulcast_layers[2].active = false;
5602
5603 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5604 kMaxPayloadLength);
5605 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5606
5607 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5608 // Frame should not be dropped.
5609 WaitForEncodedFrame(2);
5610
5611 // Trigger HD "singlecast"
5612 video_encoder_config.simulcast_layers[0].active = false;
5613 video_encoder_config.simulcast_layers[1].active = false;
5614 video_encoder_config.simulcast_layers[2].active = true;
5615
5616 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5617 kMaxPayloadLength);
5618 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5619
5620 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5621 // Frame should be dropped because of initial frame drop.
5622 ExpectDroppedFrame();
5623
5624 // Expect the sink_wants to specify a scaled frame.
5625 EXPECT_TRUE_WAIT(
5626 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5627 video_stream_encoder_->Stop();
5628}
5629
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005630TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005631 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005632 // Set simulcast.
5633 ResetEncoder("VP9", 1, 1, 3, false);
5634 fake_encoder_.SetQualityScaling(true);
5635 const int kWidth = 1280;
5636 const int kHeight = 720;
5637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005638 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005639 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5640 // Frame should not be dropped.
5641 WaitForEncodedFrame(1);
5642
5643 // Trigger QVGA "singlecast"
5644 // Update the config.
5645 VideoEncoderConfig video_encoder_config;
5646 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5647 &video_encoder_config);
5648 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5649 vp9_settings.numberOfSpatialLayers = 3;
5650 // Since only one layer is active - automatic resize should be enabled.
5651 vp9_settings.automaticResizeOn = true;
5652 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005653 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005654 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005655 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005656 video_encoder_config.content_type =
5657 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005658 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005659 // which SVC layers are active.
5660 video_encoder_config.simulcast_layers.resize(3);
5661
5662 video_encoder_config.simulcast_layers[0].active = true;
5663 video_encoder_config.simulcast_layers[1].active = false;
5664 video_encoder_config.simulcast_layers[2].active = false;
5665
5666 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5667 kMaxPayloadLength);
5668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5669
5670 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5671 // Frame should not be dropped.
5672 WaitForEncodedFrame(2);
5673
5674 // Trigger HD "singlecast"
5675 video_encoder_config.simulcast_layers[0].active = false;
5676 video_encoder_config.simulcast_layers[1].active = false;
5677 video_encoder_config.simulcast_layers[2].active = true;
5678
5679 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5680 kMaxPayloadLength);
5681 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5682
5683 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5684 // Frame should be dropped because of initial frame drop.
5685 ExpectDroppedFrame();
5686
5687 // Expect the sink_wants to specify a scaled frame.
5688 EXPECT_TRUE_WAIT(
5689 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5690 video_stream_encoder_->Stop();
5691}
5692
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005693TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005694 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5695 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5696 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5697 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5698 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5699 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5700 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5701 fake_encoder_.SetResolutionBitrateLimits(
5702 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5703
5704 VideoEncoderConfig video_encoder_config;
5705 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5706 &video_encoder_config);
5707 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5708 vp9_settings.numberOfSpatialLayers = 3;
5709 // Since only one layer is active - automatic resize should be enabled.
5710 vp9_settings.automaticResizeOn = true;
5711 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005712 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005713 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005714 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005715 video_encoder_config.content_type =
5716 VideoEncoderConfig::ContentType::kRealtimeVideo;
5717 // Simulcast layers are used to indicate which spatial layers are active.
5718 video_encoder_config.simulcast_layers.resize(3);
5719 video_encoder_config.simulcast_layers[0].active = false;
5720 video_encoder_config.simulcast_layers[1].active = true;
5721 video_encoder_config.simulcast_layers[2].active = false;
5722
5723 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5724 kMaxPayloadLength);
5725 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5726
5727 // The encoder bitrate limits for 360p should be used.
5728 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5729 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005730 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5731 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5732 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5733 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5734 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5735 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005736 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005737 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005738 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005739 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005740
5741 // The encoder bitrate limits for 270p should be used.
5742 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5743 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005744 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5745 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5746 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5747 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5748 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5749 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005750 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005751 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005752 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005753 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005754
5755 video_stream_encoder_->Stop();
5756}
5757
5758TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005759 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5760 VideoEncoderConfig video_encoder_config;
5761 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5762 &video_encoder_config);
5763 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5764 vp9_settings.numberOfSpatialLayers = 3;
5765 // Since only one layer is active - automatic resize should be enabled.
5766 vp9_settings.automaticResizeOn = true;
5767 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005768 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005769 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005770 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005771 video_encoder_config.content_type =
5772 VideoEncoderConfig::ContentType::kRealtimeVideo;
5773 // Simulcast layers are used to indicate which spatial layers are active.
5774 video_encoder_config.simulcast_layers.resize(3);
5775 video_encoder_config.simulcast_layers[0].active = false;
5776 video_encoder_config.simulcast_layers[1].active = true;
5777 video_encoder_config.simulcast_layers[2].active = false;
5778
5779 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5780 kMaxPayloadLength);
5781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5782
5783 // The default bitrate limits for 360p should be used.
5784 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005785 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5786 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005787 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5788 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005789 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5790 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5791 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5792 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5793 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5794 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005795 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005796 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005797 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005798 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005799
5800 // The default bitrate limits for 270p should be used.
5801 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005802 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5803 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005804 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5805 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005806 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5807 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5808 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5809 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5810 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5811 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005812 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005813 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005814 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005815 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005816
5817 video_stream_encoder_->Stop();
5818}
5819
5820TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5821 webrtc::test::ScopedFieldTrials field_trials(
5822 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5823 VideoEncoderConfig video_encoder_config;
5824 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5825 &video_encoder_config);
5826 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5827 vp9_settings.numberOfSpatialLayers = 3;
5828 // Since only one layer is active - automatic resize should be enabled.
5829 vp9_settings.automaticResizeOn = true;
5830 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005831 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005832 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005833 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005834 video_encoder_config.content_type =
5835 VideoEncoderConfig::ContentType::kRealtimeVideo;
5836 // Simulcast layers are used to indicate which spatial layers are active.
5837 video_encoder_config.simulcast_layers.resize(3);
5838 video_encoder_config.simulcast_layers[0].active = false;
5839 video_encoder_config.simulcast_layers[1].active = true;
5840 video_encoder_config.simulcast_layers[2].active = false;
5841
5842 // Reset encoder for field trials to take effect.
5843 ConfigureEncoder(video_encoder_config.Copy());
5844
5845 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5846 kMaxPayloadLength);
5847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5848
5849 // The default bitrate limits for 360p should not be used.
5850 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005851 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5852 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005853 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5854 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005855 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5856 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5857 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5858 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5859 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5860 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005861 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005862 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005863
5864 video_stream_encoder_->Stop();
5865}
5866
5867TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5868 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5869 /*num_spatial_layers=*/1, /*screenshare=*/false);
5870
5871 // The default singlecast bitrate limits for 720p should not be used.
5872 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005873 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5874 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005875 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5876 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005877 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5878 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5879 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5880 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5881 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5882 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005883 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005884 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005885
5886 video_stream_encoder_->Stop();
5887}
5888
5889TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005890 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5891 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5892 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5893 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5894 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5895 fake_encoder_.SetResolutionBitrateLimits(
5896 {kEncoderLimits180p, kEncoderLimits720p});
5897
5898 VideoEncoderConfig video_encoder_config;
5899 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5900 &video_encoder_config);
5901 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5902 vp9_settings.numberOfSpatialLayers = 3;
5903 // Since only one layer is active - automatic resize should be enabled.
5904 vp9_settings.automaticResizeOn = true;
5905 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005906 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005907 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005908 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005909 video_encoder_config.content_type =
5910 VideoEncoderConfig::ContentType::kRealtimeVideo;
5911 // Simulcast layers are used to indicate which spatial layers are active.
5912 video_encoder_config.simulcast_layers.resize(3);
5913 video_encoder_config.simulcast_layers[0].active = true;
5914 video_encoder_config.simulcast_layers[1].active = false;
5915 video_encoder_config.simulcast_layers[2].active = false;
5916
5917 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5918 kMaxPayloadLength);
5919 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5920
5921 // Limits not applied on lowest stream, limits for 180p should not be used.
5922 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5923 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005924 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5925 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5926 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5927 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5928 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5929 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005930 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005931 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005932 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005933 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005934
5935 video_stream_encoder_->Stop();
5936}
5937
5938TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005939 InitialFrameDropActivatesWhenResolutionIncreases) {
5940 const int kWidth = 640;
5941 const int kHeight = 360;
5942
5943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005944 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005945 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5946 // Frame should not be dropped.
5947 WaitForEncodedFrame(1);
5948
5949 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005950 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005951 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5952 // Frame should not be dropped, bitrate not too low for frame.
5953 WaitForEncodedFrame(2);
5954
5955 // Incoming resolution increases.
5956 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5957 // Expect to drop this frame, bitrate too low for frame.
5958 ExpectDroppedFrame();
5959
5960 // Expect the sink_wants to specify a scaled frame.
5961 EXPECT_TRUE_WAIT(
5962 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5963 video_stream_encoder_->Stop();
5964}
5965
5966TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5967 const int kWidth = 640;
5968 const int kHeight = 360;
5969 // So that quality scaling doesn't happen by itself.
5970 fake_encoder_.SetQp(kQpHigh);
5971
5972 AdaptingFrameForwarder source(&time_controller_);
5973 source.set_adaptation_enabled(true);
5974 video_stream_encoder_->SetSource(
5975 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5976
5977 int timestamp = 1;
5978
5979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005980 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005981 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5982 WaitForEncodedFrame(timestamp);
5983 timestamp += 9000;
5984 // Long pause to disable all first BWE drop logic.
5985 AdvanceTime(TimeDelta::Millis(1000));
5986
5987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005988 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005989 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5990 // Not dropped frame, as initial frame drop is disabled by now.
5991 WaitForEncodedFrame(timestamp);
5992 timestamp += 9000;
5993 AdvanceTime(TimeDelta::Millis(100));
5994
5995 // Quality adaptation down.
5996 video_stream_encoder_->TriggerQualityLow();
5997
5998 // Adaptation has an effect.
5999 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6000 5000);
6001
6002 // Frame isn't dropped as initial frame dropper is disabled.
6003 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6004 WaitForEncodedFrame(timestamp);
6005 timestamp += 9000;
6006 AdvanceTime(TimeDelta::Millis(100));
6007
6008 // Quality adaptation up.
6009 video_stream_encoder_->TriggerQualityHigh();
6010
6011 // Adaptation has an effect.
6012 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6013 5000);
6014
6015 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6016 // Frame should not be dropped, as initial framedropper is off.
6017 WaitForEncodedFrame(timestamp);
6018
6019 video_stream_encoder_->Stop();
6020}
6021
Åsa Persson7f354f82021-02-04 15:52:15 +01006022TEST_F(VideoStreamEncoderTest,
6023 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6024 const int kMinStartBps360p = 222000;
6025 fake_encoder_.SetResolutionBitrateLimits(
6026 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6027 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6028 800000)});
6029
6030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6031 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6032 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6033 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6034 0, 0, 0);
6035 // Frame should not be dropped, bitrate not too low for frame.
6036 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6037 WaitForEncodedFrame(1);
6038
6039 // Incoming resolution increases, initial frame drop activates.
6040 // Frame should be dropped, link allocation too low for frame.
6041 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6042 ExpectDroppedFrame();
6043
6044 // Expect sink_wants to specify a scaled frame.
6045 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6046 5000);
6047 video_stream_encoder_->Stop();
6048}
6049
6050TEST_F(VideoStreamEncoderTest,
6051 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6052 const int kMinStartBps360p = 222000;
6053 fake_encoder_.SetResolutionBitrateLimits(
6054 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6055 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6056 800000)});
6057
6058 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6059 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6060 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6061 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6062 0, 0, 0);
6063 // Frame should not be dropped, bitrate not too low for frame.
6064 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6065 WaitForEncodedFrame(1);
6066
6067 // Incoming resolution increases, initial frame drop activates.
6068 // Frame should be dropped, link allocation not too low for frame.
6069 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6070 WaitForEncodedFrame(2);
6071
6072 video_stream_encoder_->Stop();
6073}
6074
Åsa Perssone644a032019-11-08 15:56:00 +01006075TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6076 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006077 "WebRTC-Video-QualityRampupSettings/"
6078 "min_pixels:921600,min_duration_ms:2000/");
6079
6080 const int kWidth = 1280;
6081 const int kHeight = 720;
6082 const int kFps = 10;
6083 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006084
6085 // Reset encoder for field trials to take effect.
6086 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006087 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006088 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006089 ConfigureEncoder(std::move(config));
6090 fake_encoder_.SetQp(kQpLow);
6091
6092 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006093 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006094 source.set_adaptation_enabled(true);
6095 video_stream_encoder_->SetSource(&source,
6096 DegradationPreference::MAINTAIN_FRAMERATE);
6097
6098 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006099 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006101 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006102
6103 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006104 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006105 int64_t timestamp_ms = kFrameIntervalMs;
6106 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6107 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006108 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6109 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006110
6111 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006112 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6113 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006114
Artem Titovab30d722021-07-27 16:22:11 +02006115 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006116 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006117 for (size_t i = 1; i <= 10; i++) {
6118 timestamp_ms += kFrameIntervalMs;
6119 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6120 WaitForEncodedFrame(timestamp_ms);
6121 }
Åsa Persson06defc42021-09-10 15:28:48 +02006122
6123 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6124 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6125 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6126 timestamp_ms += kFrameIntervalMs;
6127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6128 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006129 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6130 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6131
Åsa Persson06defc42021-09-10 15:28:48 +02006132 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006133 timestamp_ms += kFrameIntervalMs;
6134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6135 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006136 // The ramp-up code involves the adaptation queue, give it time to execute.
6137 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006138 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006139 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006140
6141 // Frame should not be adapted.
6142 timestamp_ms += kFrameIntervalMs;
6143 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6144 WaitForEncodedFrame(kWidth, kHeight);
6145 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6146
6147 video_stream_encoder_->Stop();
6148}
6149
mflodmancc3d4422017-08-03 08:27:51 -07006150TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006151 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006152 webrtc::test::ScopedFieldTrials field_trials(
6153 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006154 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006155 source.set_adaptation_enabled(true);
6156 video_stream_encoder_->SetSource(&source,
6157 DegradationPreference::MAINTAIN_FRAMERATE);
6158 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006159 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006160 fake_encoder_.SetQp(kQpHigh + 1);
6161 const int kWidth = 1280;
6162 const int kHeight = 720;
6163 const int64_t kFrameIntervalMs = 100;
6164 int64_t timestamp_ms = kFrameIntervalMs;
6165 for (size_t i = 1; i <= 100; i++) {
6166 timestamp_ms += kFrameIntervalMs;
6167 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6168 WaitForEncodedFrame(timestamp_ms);
6169 }
6170 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6171 // for the first time.
6172 // TODO(eshr): We should avoid these waits by using threads with simulated
6173 // time.
6174 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6175 2000 * 2.5 * 2);
6176 timestamp_ms += kFrameIntervalMs;
6177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6178 WaitForEncodedFrame(timestamp_ms);
6179 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6180 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6181 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6182
6183 // Disable Quality scaling by turning off scaler on the encoder and
6184 // reconfiguring.
6185 fake_encoder_.SetQualityScaling(false);
6186 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6187 kMaxPayloadLength);
6188 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006189 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006190 // Since we turned off the quality scaler, the adaptations made by it are
6191 // removed.
6192 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6194
6195 video_stream_encoder_->Stop();
6196}
6197
6198TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006199 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6200 const int kTooSmallWidth = 10;
6201 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006203 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006204
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006205 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006206 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006207 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006208 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006209 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006210 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6211
6212 // Trigger adapt down, too small frame, expect no change.
6213 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006214 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006215 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006216 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006217 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6218 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6219
mflodmancc3d4422017-08-03 08:27:51 -07006220 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006221}
6222
mflodmancc3d4422017-08-03 08:27:51 -07006223TEST_F(VideoStreamEncoderTest,
6224 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006225 const int kTooSmallWidth = 10;
6226 const int kTooSmallHeight = 10;
6227 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006229 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006230
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006231 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006232 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006233 video_stream_encoder_->SetSource(&source,
6234 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006235 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6237 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6238
6239 // Trigger adapt down, expect limited framerate.
6240 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006241 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006242 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006243 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6245 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6246 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6247
6248 // Trigger adapt down, too small frame, expect no change.
6249 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006250 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006251 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006252 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006253 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6254 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6255 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6256
mflodmancc3d4422017-08-03 08:27:51 -07006257 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006258}
6259
mflodmancc3d4422017-08-03 08:27:51 -07006260TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006261 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006263 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006264 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006265 const int kFrameWidth = 1280;
6266 const int kFrameHeight = 720;
6267 video_source_.IncomingCapturedFrame(
6268 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006269 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006270 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006271}
6272
sprangb1ca0732017-02-01 08:38:12 -08006273// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006274TEST_F(VideoStreamEncoderTest,
6275 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006277 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006278
6279 const int kFrameWidth = 1280;
6280 const int kFrameHeight = 720;
6281 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006282 // requested by
6283 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006284 video_source_.set_adaptation_enabled(true);
6285
6286 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006287 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006288 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006289
6290 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006292 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006293 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006294 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006295
asaperssonfab67072017-04-04 05:51:49 -07006296 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006297 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006298 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006299 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006300 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006301
mflodmancc3d4422017-08-03 08:27:51 -07006302 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006303}
sprangfe627f32017-03-29 08:24:59 -07006304
mflodmancc3d4422017-08-03 08:27:51 -07006305TEST_F(VideoStreamEncoderTest,
6306 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006307 const int kFrameWidth = 1280;
6308 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006309
Henrik Boström381d1092020-05-12 18:49:07 +02006310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006311 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006312 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006313 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006314 video_source_.set_adaptation_enabled(true);
6315
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006316 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006317
6318 video_source_.IncomingCapturedFrame(
6319 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006320 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006321
6322 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006323 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006324
6325 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006326 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006327 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006328 video_source_.IncomingCapturedFrame(
6329 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006330 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006331 }
6332
6333 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006334 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006335 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006336 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006337 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006338 video_source_.IncomingCapturedFrame(
6339 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006340 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006341 ++num_frames_dropped;
6342 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006343 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006344 }
6345 }
6346
sprang4847ae62017-06-27 07:06:52 -07006347 // Add some slack to account for frames dropped by the frame dropper.
6348 const int kErrorMargin = 1;
6349 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006350 kErrorMargin);
6351
6352 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006353 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006354 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006355 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006356 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006357 video_source_.IncomingCapturedFrame(
6358 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006359 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006360 ++num_frames_dropped;
6361 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006362 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006363 }
6364 }
sprang4847ae62017-06-27 07:06:52 -07006365 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006366 kErrorMargin);
6367
6368 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006369 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006370 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006371 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006372 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006373 video_source_.IncomingCapturedFrame(
6374 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006375 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006376 ++num_frames_dropped;
6377 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006378 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006379 }
6380 }
sprang4847ae62017-06-27 07:06:52 -07006381 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006382 kErrorMargin);
6383
6384 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006385 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006386 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006387 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006388 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006389 video_source_.IncomingCapturedFrame(
6390 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006391 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006392 ++num_frames_dropped;
6393 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006394 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006395 }
6396 }
6397 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6398
mflodmancc3d4422017-08-03 08:27:51 -07006399 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006400}
6401
mflodmancc3d4422017-08-03 08:27:51 -07006402TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006403 const int kFramerateFps = 5;
6404 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006405 const int kFrameWidth = 1280;
6406 const int kFrameHeight = 720;
6407
sprang4847ae62017-06-27 07:06:52 -07006408 // Reconfigure encoder with two temporal layers and screensharing, which will
6409 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006410 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006411
Henrik Boström381d1092020-05-12 18:49:07 +02006412 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006413 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006414 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006415 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006416 video_source_.set_adaptation_enabled(true);
6417
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006418 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006419
6420 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006421 rtc::VideoSinkWants last_wants;
6422 do {
6423 last_wants = video_source_.sink_wants();
6424
sprangc5d62e22017-04-02 23:53:04 -07006425 // Insert frames to get a new fps estimate...
6426 for (int j = 0; j < kFramerateFps; ++j) {
6427 video_source_.IncomingCapturedFrame(
6428 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006429 if (video_source_.last_sent_width()) {
6430 sink_.WaitForEncodedFrame(timestamp_ms);
6431 }
sprangc5d62e22017-04-02 23:53:04 -07006432 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006433 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006434 }
6435 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006436 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006437 } while (video_source_.sink_wants().max_framerate_fps <
6438 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006439
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006440 EXPECT_THAT(video_source_.sink_wants(),
6441 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006442
mflodmancc3d4422017-08-03 08:27:51 -07006443 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006444}
asaperssonf7e294d2017-06-13 23:25:22 -07006445
mflodmancc3d4422017-08-03 08:27:51 -07006446TEST_F(VideoStreamEncoderTest,
6447 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006448 const int kWidth = 1280;
6449 const int kHeight = 720;
6450 const int64_t kFrameIntervalMs = 150;
6451 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006453 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006454
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006455 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006456 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006457 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006458 video_stream_encoder_->SetSource(&source,
6459 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006460 timestamp_ms += kFrameIntervalMs;
6461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006462 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006463 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006464 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6466 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6467
6468 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006469 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006470 timestamp_ms += kFrameIntervalMs;
6471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006473 EXPECT_THAT(source.sink_wants(),
6474 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006475 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6476 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6477 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6478
6479 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006480 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006481 timestamp_ms += kFrameIntervalMs;
6482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006483 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006484 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006485 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6487 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6488
6489 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006490 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006491 timestamp_ms += kFrameIntervalMs;
6492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006493 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006494 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6497 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6498
6499 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006500 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006501 timestamp_ms += kFrameIntervalMs;
6502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006503 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006504 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6506 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6507 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6508
6509 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006510 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006511 timestamp_ms += kFrameIntervalMs;
6512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006513 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006514 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6517 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6518
6519 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006520 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006521 timestamp_ms += kFrameIntervalMs;
6522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006523 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006524 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6527 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6528
6529 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006530 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006531 timestamp_ms += kFrameIntervalMs;
6532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006533 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006534 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006535 rtc::VideoSinkWants last_wants = source.sink_wants();
6536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6538 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6539
6540 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006542 timestamp_ms += kFrameIntervalMs;
6543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006544 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006545 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6548 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6549
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006550 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006551 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006552 timestamp_ms += kFrameIntervalMs;
6553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006555 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6569
6570 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006571 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006572 timestamp_ms += kFrameIntervalMs;
6573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006574 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006575 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6578 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6579
6580 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006581 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006582 timestamp_ms += kFrameIntervalMs;
6583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006584 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006585 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6588 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6589
6590 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006591 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006594 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006595 EXPECT_THAT(source.sink_wants(), FpsMax());
6596 EXPECT_EQ(source.sink_wants().max_pixel_count,
6597 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6599 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6600 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6601
6602 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006603 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006604 timestamp_ms += kFrameIntervalMs;
6605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006606 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006607 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6610 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611
Åsa Persson30ab0152019-08-27 12:22:33 +02006612 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006613 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006614 timestamp_ms += kFrameIntervalMs;
6615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006616 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006617 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006618 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6620 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6621 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6622
6623 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006624 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006625 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006626 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6627
mflodmancc3d4422017-08-03 08:27:51 -07006628 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006629}
6630
mflodmancc3d4422017-08-03 08:27:51 -07006631TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006632 const int kWidth = 1280;
6633 const int kHeight = 720;
6634 const int64_t kFrameIntervalMs = 150;
6635 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006637 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006638
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006639 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006640 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006641 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006642 video_stream_encoder_->SetSource(&source,
6643 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006644 timestamp_ms += kFrameIntervalMs;
6645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006646 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006647 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6650 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6652 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6654
6655 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006656 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006657 timestamp_ms += kFrameIntervalMs;
6658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006659 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006660 EXPECT_THAT(source.sink_wants(),
6661 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6664 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6666 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6667 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6668
6669 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006670 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006671 timestamp_ms += kFrameIntervalMs;
6672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006673 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006674 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6677 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6679 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6680 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6681
6682 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006683 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006684 timestamp_ms += kFrameIntervalMs;
6685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006686 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006687 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006688 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006689 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6690 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6691 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6692 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6693 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6694
Evan Shrubsole64469032020-06-11 10:45:29 +02006695 // Trigger cpu adapt up, expect no change since QP is most limited.
6696 {
6697 // Store current sink wants since we expect no change and if there is no
6698 // change then last_wants() is not updated.
6699 auto previous_sink_wants = source.sink_wants();
6700 video_stream_encoder_->TriggerCpuUnderuse();
6701 timestamp_ms += kFrameIntervalMs;
6702 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6703 WaitForEncodedFrame(timestamp_ms);
6704 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6705 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6706 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6707 }
6708
6709 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6710 video_stream_encoder_->TriggerQualityHigh();
6711 timestamp_ms += kFrameIntervalMs;
6712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6713 WaitForEncodedFrame(timestamp_ms);
6714 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6715 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6717 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6719 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6720 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6721
6722 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6723 // expect increased resolution (960x540@30fps).
6724 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006725 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006726 timestamp_ms += kFrameIntervalMs;
6727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006728 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006730 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6732 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6733 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6734 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006735 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006736
Evan Shrubsole64469032020-06-11 10:45:29 +02006737 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6738 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006739 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006740 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006741 timestamp_ms += kFrameIntervalMs;
6742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006743 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006745 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6747 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6748 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6750 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006751 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006752
6753 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006754 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006755 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006756 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006757 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006758
mflodmancc3d4422017-08-03 08:27:51 -07006759 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006760}
6761
mflodmancc3d4422017-08-03 08:27:51 -07006762TEST_F(VideoStreamEncoderTest,
6763 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006764 const int kWidth = 640;
6765 const int kHeight = 360;
6766 const int kFpsLimit = 15;
6767 const int64_t kFrameIntervalMs = 150;
6768 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006770 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006771
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006772 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006773 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006774 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006775 video_stream_encoder_->SetSource(&source,
6776 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006777 timestamp_ms += kFrameIntervalMs;
6778 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006779 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006780 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6785 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6787
6788 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006789 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006790 timestamp_ms += kFrameIntervalMs;
6791 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006792 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006793 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006794 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6795 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6796 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6797 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6798 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6799 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6800
6801 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006802 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006803 timestamp_ms += kFrameIntervalMs;
6804 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006805 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006806 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006807 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006808 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6810 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6811 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6812 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6813
Evan Shrubsole64469032020-06-11 10:45:29 +02006814 // Trigger cpu adapt up, expect no change because quality is most limited.
6815 {
6816 auto previous_sink_wants = source.sink_wants();
6817 // Store current sink wants since we expect no change ind if there is no
6818 // change then last__wants() is not updated.
6819 video_stream_encoder_->TriggerCpuUnderuse();
6820 timestamp_ms += kFrameIntervalMs;
6821 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6822 WaitForEncodedFrame(timestamp_ms);
6823 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6824 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6825 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6826 }
6827
6828 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6829 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006830 timestamp_ms += kFrameIntervalMs;
6831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006832 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006833 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006834 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6835 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6836 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006837 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6838 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6839 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006840
Evan Shrubsole64469032020-06-11 10:45:29 +02006841 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006842 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006843 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006844 timestamp_ms += kFrameIntervalMs;
6845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006846 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006847 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6850 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6851 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6852 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006853 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006854
6855 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006856 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006857 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006858 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006859 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006860
mflodmancc3d4422017-08-03 08:27:51 -07006861 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006862}
6863
mflodmancc3d4422017-08-03 08:27:51 -07006864TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006865 const int kFrameWidth = 1920;
6866 const int kFrameHeight = 1080;
6867 // 3/4 of 1920.
6868 const int kAdaptedFrameWidth = 1440;
6869 // 3/4 of 1080 rounded down to multiple of 4.
6870 const int kAdaptedFrameHeight = 808;
6871 const int kFramerate = 24;
6872
Henrik Boström381d1092020-05-12 18:49:07 +02006873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006874 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006875 // Trigger reconfigure encoder (without resetting the entire instance).
6876 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006877 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6878 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006879 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006880 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006881 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006882 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006883 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006884 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006885
6886 video_source_.set_adaptation_enabled(true);
6887
6888 video_source_.IncomingCapturedFrame(
6889 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006890 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006891
6892 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006893 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006894 video_source_.IncomingCapturedFrame(
6895 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006896 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006897
mflodmancc3d4422017-08-03 08:27:51 -07006898 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006899}
6900
mflodmancc3d4422017-08-03 08:27:51 -07006901TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006902 const int kFrameWidth = 1280;
6903 const int kFrameHeight = 720;
6904 const int kLowFps = 2;
6905 const int kHighFps = 30;
6906
Henrik Boström381d1092020-05-12 18:49:07 +02006907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006908 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006909
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006910 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006911 max_framerate_ = kLowFps;
6912
6913 // Insert 2 seconds of 2fps video.
6914 for (int i = 0; i < kLowFps * 2; ++i) {
6915 video_source_.IncomingCapturedFrame(
6916 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6917 WaitForEncodedFrame(timestamp_ms);
6918 timestamp_ms += 1000 / kLowFps;
6919 }
6920
6921 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006924 video_source_.IncomingCapturedFrame(
6925 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6926 WaitForEncodedFrame(timestamp_ms);
6927 timestamp_ms += 1000 / kLowFps;
6928
6929 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6930
6931 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006932 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006933 const int kFrameIntervalMs = 1000 / kHighFps;
6934 max_framerate_ = kHighFps;
6935 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6936 video_source_.IncomingCapturedFrame(
6937 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6938 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6939 // be dropped if the encoder hans't been updated with the new higher target
6940 // framerate yet, causing it to overshoot the target bitrate and then
6941 // suffering the wrath of the media optimizer.
6942 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6943 timestamp_ms += kFrameIntervalMs;
6944 }
6945
6946 // Don expect correct measurement just yet, but it should be higher than
6947 // before.
6948 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6949
mflodmancc3d4422017-08-03 08:27:51 -07006950 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006951}
6952
mflodmancc3d4422017-08-03 08:27:51 -07006953TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006954 const int kFrameWidth = 1280;
6955 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006956 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006957 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006958 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006959
Henrik Boström381d1092020-05-12 18:49:07 +02006960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006961 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006962 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006963
6964 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006965 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006966 video_source_.IncomingCapturedFrame(
6967 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6968 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006969 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006970
6971 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006972 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006973 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006974
6975 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006976 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006977 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006978
Per Kjellanderdcef6412020-10-07 15:09:05 +02006979 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006980 video_source_.IncomingCapturedFrame(
6981 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6982 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006983 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006984
mflodmancc3d4422017-08-03 08:27:51 -07006985 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006986}
ilnik6b826ef2017-06-16 06:53:48 -07006987
Niels Möller4db138e2018-04-19 09:04:13 +02006988TEST_F(VideoStreamEncoderTest,
6989 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6990 const int kFrameWidth = 1280;
6991 const int kFrameHeight = 720;
6992 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(1, kFrameWidth, kFrameHeight));
6997 WaitForEncodedFrame(1);
6998 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6999 .low_encode_usage_threshold_percent,
7000 default_options.low_encode_usage_threshold_percent);
7001 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7002 .high_encode_usage_threshold_percent,
7003 default_options.high_encode_usage_threshold_percent);
7004 video_stream_encoder_->Stop();
7005}
7006
7007TEST_F(VideoStreamEncoderTest,
7008 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7009 const int kFrameWidth = 1280;
7010 const int kFrameHeight = 720;
7011 CpuOveruseOptions hardware_options;
7012 hardware_options.low_encode_usage_threshold_percent = 150;
7013 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007014 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007015
Henrik Boström381d1092020-05-12 18:49:07 +02007016 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007017 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007018 video_source_.IncomingCapturedFrame(
7019 CreateFrame(1, kFrameWidth, kFrameHeight));
7020 WaitForEncodedFrame(1);
7021 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7022 .low_encode_usage_threshold_percent,
7023 hardware_options.low_encode_usage_threshold_percent);
7024 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7025 .high_encode_usage_threshold_percent,
7026 hardware_options.high_encode_usage_threshold_percent);
7027 video_stream_encoder_->Stop();
7028}
7029
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007030TEST_F(VideoStreamEncoderTest,
7031 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7032 const int kFrameWidth = 1280;
7033 const int kFrameHeight = 720;
7034
7035 const CpuOveruseOptions default_options;
7036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007037 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007038 video_source_.IncomingCapturedFrame(
7039 CreateFrame(1, kFrameWidth, kFrameHeight));
7040 WaitForEncodedFrame(1);
7041 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7042 .low_encode_usage_threshold_percent,
7043 default_options.low_encode_usage_threshold_percent);
7044 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7045 .high_encode_usage_threshold_percent,
7046 default_options.high_encode_usage_threshold_percent);
7047
7048 CpuOveruseOptions hardware_options;
7049 hardware_options.low_encode_usage_threshold_percent = 150;
7050 hardware_options.high_encode_usage_threshold_percent = 200;
7051 fake_encoder_.SetIsHardwareAccelerated(true);
7052
7053 video_source_.IncomingCapturedFrame(
7054 CreateFrame(2, kFrameWidth, kFrameHeight));
7055 WaitForEncodedFrame(2);
7056
7057 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7058 .low_encode_usage_threshold_percent,
7059 hardware_options.low_encode_usage_threshold_percent);
7060 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7061 .high_encode_usage_threshold_percent,
7062 hardware_options.high_encode_usage_threshold_percent);
7063
7064 video_stream_encoder_->Stop();
7065}
7066
Niels Möller6bb5ab92019-01-11 11:11:10 +01007067TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7068 const int kFrameWidth = 320;
7069 const int kFrameHeight = 240;
7070 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007071 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007072 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7073
Henrik Boström381d1092020-05-12 18:49:07 +02007074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007075 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007076
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007077 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007078 max_framerate_ = kFps;
7079
7080 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7081 fake_encoder_.SimulateOvershoot(1.0);
7082 int num_dropped = 0;
7083 for (int i = 0; i < kNumFramesInRun; ++i) {
7084 video_source_.IncomingCapturedFrame(
7085 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7086 // Wait up to two frame durations for a frame to arrive.
7087 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7088 ++num_dropped;
7089 }
7090 timestamp_ms += 1000 / kFps;
7091 }
7092
Erik Språnga8d48ab2019-02-08 14:17:40 +01007093 // Framerate should be measured to be near the expected target rate.
7094 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7095
7096 // Frame drops should be within 5% of expected 0%.
7097 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007098
7099 // Make encoder produce frames at double the expected bitrate during 3 seconds
7100 // of video, verify number of drops. Rate needs to be slightly changed in
7101 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007102 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007103 const RateControlSettings trials =
7104 RateControlSettings::ParseFromFieldTrials();
7105 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007106 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007107 // frame dropping since the adjuter will try to just lower the target
7108 // bitrate rather than drop frames. If network headroom can be used, it
7109 // doesn't push back as hard so we don't need quite as much overshoot.
7110 // These numbers are unfortunately a bit magical but there's not trivial
7111 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007112 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007113 }
7114 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007116 kTargetBitrate + DataRate::KilobitsPerSec(1),
7117 kTargetBitrate + DataRate::KilobitsPerSec(1),
7118 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007119 num_dropped = 0;
7120 for (int i = 0; i < kNumFramesInRun; ++i) {
7121 video_source_.IncomingCapturedFrame(
7122 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7123 // Wait up to two frame durations for a frame to arrive.
7124 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7125 ++num_dropped;
7126 }
7127 timestamp_ms += 1000 / kFps;
7128 }
7129
Henrik Boström381d1092020-05-12 18:49:07 +02007130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007131 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007132
7133 // Target framerate should be still be near the expected target, despite
7134 // the frame drops.
7135 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7136
7137 // Frame drops should be within 5% of expected 50%.
7138 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007139
7140 video_stream_encoder_->Stop();
7141}
7142
7143TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7144 const int kFrameWidth = 320;
7145 const int kFrameHeight = 240;
7146 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007147 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007148
7149 ASSERT_GT(max_framerate_, kActualInputFps);
7150
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007151 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007152 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007153 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007154 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007155
7156 // Insert 3 seconds of video, with an input fps lower than configured max.
7157 for (int i = 0; i < kActualInputFps * 3; ++i) {
7158 video_source_.IncomingCapturedFrame(
7159 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7160 // Wait up to two frame durations for a frame to arrive.
7161 WaitForEncodedFrame(timestamp_ms);
7162 timestamp_ms += 1000 / kActualInputFps;
7163 }
7164
7165 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7166
7167 video_stream_encoder_->Stop();
7168}
7169
Markus Handell9a478b52021-11-18 16:07:01 +01007170TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007171 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007172 test::FrameForwarder source;
7173 video_stream_encoder_->SetSource(&source,
7174 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007176 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007177
Markus Handell9a478b52021-11-18 16:07:01 +01007178 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007179 WaitForEncodedFrame(1);
7180 // On the very first frame full update should be forced.
7181 rect = fake_encoder_.GetLastUpdateRect();
7182 EXPECT_EQ(rect.offset_x, 0);
7183 EXPECT_EQ(rect.offset_y, 0);
7184 EXPECT_EQ(rect.height, codec_height_);
7185 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007186 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7187 // scheduled for processing during encoder queue processing of frame 2.
7188 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7189 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007190 WaitForEncodedFrame(3);
7191 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7192 rect = fake_encoder_.GetLastUpdateRect();
7193 EXPECT_EQ(rect.offset_x, 1);
7194 EXPECT_EQ(rect.offset_y, 0);
7195 EXPECT_EQ(rect.width, 10);
7196 EXPECT_EQ(rect.height, 1);
7197
Markus Handell9a478b52021-11-18 16:07:01 +01007198 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007199 WaitForEncodedFrame(4);
7200 // Previous frame was encoded, so no accumulation should happen.
7201 rect = fake_encoder_.GetLastUpdateRect();
7202 EXPECT_EQ(rect.offset_x, 0);
7203 EXPECT_EQ(rect.offset_y, 0);
7204 EXPECT_EQ(rect.width, 1);
7205 EXPECT_EQ(rect.height, 1);
7206
7207 video_stream_encoder_->Stop();
7208}
7209
Erik Språngd7329ca2019-02-21 21:19:53 +01007210TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007211 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007212 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007213
7214 // First frame is always keyframe.
7215 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7216 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007217 EXPECT_THAT(
7218 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007219 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007220
7221 // Insert delta frame.
7222 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7223 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007224 EXPECT_THAT(
7225 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007226 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007227
7228 // Request next frame be a key-frame.
7229 video_stream_encoder_->SendKeyFrame();
7230 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7231 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007232 EXPECT_THAT(
7233 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007234 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007235
7236 video_stream_encoder_->Stop();
7237}
7238
7239TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7240 // Setup simulcast with three streams.
7241 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007243 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7244 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007245 // Wait for all three layers before triggering event.
7246 sink_.SetNumExpectedLayers(3);
7247
7248 // First frame is always keyframe.
7249 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7250 WaitForEncodedFrame(1);
7251 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007252 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7253 VideoFrameType::kVideoFrameKey,
7254 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007255
7256 // Insert delta frame.
7257 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7258 WaitForEncodedFrame(2);
7259 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007260 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7261 VideoFrameType::kVideoFrameDelta,
7262 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007263
7264 // Request next frame be a key-frame.
7265 // Only first stream is configured to produce key-frame.
7266 video_stream_encoder_->SendKeyFrame();
7267 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7268 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007269
7270 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7271 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007272 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007273 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007274 VideoFrameType::kVideoFrameKey,
7275 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007276
7277 video_stream_encoder_->Stop();
7278}
7279
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007280TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007281 // SPS contains VUI with restrictions on the maximum number of reordered
7282 // pictures, there is no need to rewrite the bitstream to enable faster
7283 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007284 ResetEncoder("H264", 1, 1, 1, false);
7285
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007288 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007289
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007290 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007291 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007292
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007293 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7294 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007295
7296 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007297 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007298
7299 video_stream_encoder_->Stop();
7300}
7301
7302TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007303 // SPS does not contain VUI, the bitstream is will be rewritten with added
7304 // VUI with restrictions on the maximum number of reordered pictures to
7305 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007306 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7307 0x00, 0x00, 0x03, 0x03, 0xF4,
7308 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007309 ResetEncoder("H264", 1, 1, 1, false);
7310
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007312 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007314
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007315 fake_encoder_.SetEncodedImageData(
7316 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007317
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007318 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7319 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007320
7321 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007322 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007323
7324 video_stream_encoder_->Stop();
7325}
7326
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007327TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7328 const int kFrameWidth = 1280;
7329 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007330 const DataRate kTargetBitrate =
7331 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007332
Henrik Boström381d1092020-05-12 18:49:07 +02007333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007334 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007335 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7336
7337 // Insert a first video frame. It should be dropped because of downscale in
7338 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007339 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007340 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7341 frame.set_rotation(kVideoRotation_270);
7342 video_source_.IncomingCapturedFrame(frame);
7343
7344 ExpectDroppedFrame();
7345
7346 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007347 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007348 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7349 frame.set_rotation(kVideoRotation_90);
7350 video_source_.IncomingCapturedFrame(frame);
7351
7352 WaitForEncodedFrame(timestamp_ms);
7353 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7354
7355 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007356 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007357 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7358 frame.set_rotation(kVideoRotation_180);
7359 video_source_.IncomingCapturedFrame(frame);
7360
7361 WaitForEncodedFrame(timestamp_ms);
7362 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7363
7364 video_stream_encoder_->Stop();
7365}
7366
Erik Språng5056af02019-09-02 15:53:11 +02007367TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7368 const int kFrameWidth = 320;
7369 const int kFrameHeight = 180;
7370
7371 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007373 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7374 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7375 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007376 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007377 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007378 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007379
7380 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007381 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007382 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7383 frame.set_rotation(kVideoRotation_270);
7384 video_source_.IncomingCapturedFrame(frame);
7385 WaitForEncodedFrame(timestamp_ms);
7386
7387 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007388 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007389 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7390 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007391 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007392 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007393 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007394 /*link_allocation=*/target_rate,
7395 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007396 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007397 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007398 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7399
7400 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7401 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7402 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007403 DataRate allocation_sum =
7404 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007405 EXPECT_EQ(min_rate, allocation_sum);
7406 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7407
7408 video_stream_encoder_->Stop();
7409}
7410
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007411TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007412 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007413 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007414 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007415 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007416 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7417 WaitForEncodedFrame(1);
7418
7419 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7420 ASSERT_TRUE(prev_rate_settings.has_value());
7421 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7422 kDefaultFramerate);
7423
7424 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7425 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7426 timestamp_ms += 1000 / kDefaultFramerate;
7427 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7428 WaitForEncodedFrame(timestamp_ms);
7429 }
7430 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7431 kDefaultFramerate);
7432 // Capture larger frame to trigger a reconfigure.
7433 codec_height_ *= 2;
7434 codec_width_ *= 2;
7435 timestamp_ms += 1000 / kDefaultFramerate;
7436 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7437 WaitForEncodedFrame(timestamp_ms);
7438
7439 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7440 auto current_rate_settings =
7441 fake_encoder_.GetAndResetLastRateControlSettings();
7442 // Ensure we have actually reconfigured twice
7443 // The rate settings should have been set again even though
7444 // they haven't changed.
7445 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007446 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007447
7448 video_stream_encoder_->Stop();
7449}
7450
philipeld9cc8c02019-09-16 14:53:40 +02007451struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007452 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7453 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7454 MOCK_METHOD(void,
7455 RequestEncoderSwitch,
7456 (const webrtc::SdpVideoFormat& format),
7457 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007458};
7459
philipel9b058032020-02-10 11:30:00 +01007460TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7461 constexpr int kDontCare = 100;
7462 StrictMock<MockEncoderSelector> encoder_selector;
7463 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7464 &fake_encoder_, &encoder_selector);
7465 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7466
7467 // Reset encoder for new configuration to take effect.
7468 ConfigureEncoder(video_encoder_config_.Copy());
7469
7470 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7471
7472 video_source_.IncomingCapturedFrame(
7473 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007474 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007475 video_stream_encoder_->Stop();
7476
7477 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7478 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007479 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7480 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007481 video_stream_encoder_.reset();
7482}
7483
7484TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7485 constexpr int kDontCare = 100;
7486
7487 NiceMock<MockEncoderSelector> encoder_selector;
7488 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7489 video_send_config_.encoder_settings.encoder_switch_request_callback =
7490 &switch_callback;
7491 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7492 &fake_encoder_, &encoder_selector);
7493 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7494
7495 // Reset encoder for new configuration to take effect.
7496 ConfigureEncoder(video_encoder_config_.Copy());
7497
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007498 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007499 .WillByDefault(Return(SdpVideoFormat("AV1")));
7500 EXPECT_CALL(switch_callback,
7501 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7502 Field(&SdpVideoFormat::name, "AV1"))));
7503
Henrik Boström381d1092020-05-12 18:49:07 +02007504 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007505 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7506 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7507 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007508 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007509 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007510 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007511 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007512
7513 video_stream_encoder_->Stop();
7514}
7515
7516TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7517 constexpr int kSufficientBitrateToNotDrop = 1000;
7518 constexpr int kDontCare = 100;
7519
7520 NiceMock<MockVideoEncoder> video_encoder;
7521 NiceMock<MockEncoderSelector> encoder_selector;
7522 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7523 video_send_config_.encoder_settings.encoder_switch_request_callback =
7524 &switch_callback;
7525 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7526 &video_encoder, &encoder_selector);
7527 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7528
7529 // Reset encoder for new configuration to take effect.
7530 ConfigureEncoder(video_encoder_config_.Copy());
7531
7532 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7533 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7534 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007536 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7537 /*stable_target_bitrate=*/
7538 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7539 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007540 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007541 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007542 /*cwnd_reduce_ratio=*/0);
7543
7544 ON_CALL(video_encoder, Encode(_, _))
7545 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7546 ON_CALL(encoder_selector, OnEncoderBroken())
7547 .WillByDefault(Return(SdpVideoFormat("AV2")));
7548
7549 rtc::Event encode_attempted;
7550 EXPECT_CALL(switch_callback,
7551 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7552 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7553 EXPECT_EQ(format.name, "AV2");
7554 encode_attempted.Set();
7555 });
7556
7557 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7558 encode_attempted.Wait(3000);
7559
Markus Handell28c71802021-11-08 10:11:55 +01007560 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007561
philipel9b058032020-02-10 11:30:00 +01007562 video_stream_encoder_->Stop();
7563
7564 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7565 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007566 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7567 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007568 video_stream_encoder_.reset();
7569}
7570
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007571TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007572 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007573 const int kFrameWidth = 320;
7574 const int kFrameHeight = 180;
7575
7576 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007577 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007579 /*target_bitrate=*/rate,
7580 /*stable_target_bitrate=*/rate,
7581 /*link_allocation=*/rate,
7582 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007583 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007584 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007585
7586 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007587 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007588 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7589 frame.set_rotation(kVideoRotation_270);
7590 video_source_.IncomingCapturedFrame(frame);
7591 WaitForEncodedFrame(timestamp_ms);
7592 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7593
7594 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007595 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007596 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007597 /*target_bitrate=*/new_stable_rate,
7598 /*stable_target_bitrate=*/new_stable_rate,
7599 /*link_allocation=*/rate,
7600 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007601 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007602 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007603 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7604 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7605 video_stream_encoder_->Stop();
7606}
7607
7608TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007609 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007610 const int kFrameWidth = 320;
7611 const int kFrameHeight = 180;
7612
7613 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007614 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007616 /*target_bitrate=*/rate,
7617 /*stable_target_bitrate=*/rate,
7618 /*link_allocation=*/rate,
7619 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007620 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007621 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007622
7623 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007624 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007625 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7626 frame.set_rotation(kVideoRotation_270);
7627 video_source_.IncomingCapturedFrame(frame);
7628 WaitForEncodedFrame(timestamp_ms);
7629 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7630
7631 // Set a higher target rate without changing the link_allocation. Should not
7632 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007633 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007635 /*target_bitrate=*/rate,
7636 /*stable_target_bitrate=*/new_stable_rate,
7637 /*link_allocation=*/rate,
7638 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007639 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007640 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007641 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7642 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7643 video_stream_encoder_->Stop();
7644}
7645
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007646TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7647 test::ScopedFieldTrials field_trials(
7648 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7649 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7650 const int kFramerateFps = 30;
7651 const int kWidth = 1920;
7652 const int kHeight = 1080;
7653 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7654 // Works on screenshare mode.
7655 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7656 // We rely on the automatic resolution adaptation, but we handle framerate
7657 // adaptation manually by mocking the stats proxy.
7658 video_source_.set_adaptation_enabled(true);
7659
7660 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007662 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007663 video_stream_encoder_->SetSource(&video_source_,
7664 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007665 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007666
7667 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7668 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7669
7670 // Pass enough frames with the full update to trigger animation detection.
7671 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007672 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007673 frame.set_ntp_time_ms(timestamp_ms);
7674 frame.set_timestamp_us(timestamp_ms * 1000);
7675 video_source_.IncomingCapturedFrame(frame);
7676 WaitForEncodedFrame(timestamp_ms);
7677 }
7678
7679 // Resolution should be limited.
7680 rtc::VideoSinkWants expected;
7681 expected.max_framerate_fps = kFramerateFps;
7682 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007683 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007684
7685 // Pass one frame with no known update.
7686 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007687 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007688 frame.set_ntp_time_ms(timestamp_ms);
7689 frame.set_timestamp_us(timestamp_ms * 1000);
7690 frame.clear_update_rect();
7691
7692 video_source_.IncomingCapturedFrame(frame);
7693 WaitForEncodedFrame(timestamp_ms);
7694
7695 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007696 EXPECT_THAT(video_source_.sink_wants(),
7697 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007698
7699 video_stream_encoder_->Stop();
7700}
7701
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007702TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7703 const int kWidth = 720; // 540p adapted down.
7704 const int kHeight = 405;
7705 const int kNumFrames = 3;
7706 // Works on screenshare mode.
7707 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7708 /*num_spatial_layers=*/2, /*screenshare=*/true);
7709
7710 video_source_.set_adaptation_enabled(true);
7711
7712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007713 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007714
7715 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7716
7717 // Pass enough frames with the full update to trigger animation detection.
7718 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007719 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007720 frame.set_ntp_time_ms(timestamp_ms);
7721 frame.set_timestamp_us(timestamp_ms * 1000);
7722 video_source_.IncomingCapturedFrame(frame);
7723 WaitForEncodedFrame(timestamp_ms);
7724 }
7725
7726 video_stream_encoder_->Stop();
7727}
7728
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007729TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7730 const float downscale_factors[] = {4.0, 2.0, 1.0};
7731 const int number_layers =
7732 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7733 VideoEncoderConfig config;
7734 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7735 for (int i = 0; i < number_layers; ++i) {
7736 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7737 config.simulcast_layers[i].active = true;
7738 }
7739 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007740 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007741 "VP8", /*max qp*/ 56, /*screencast*/ false,
7742 /*screenshare enabled*/ false);
7743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007744 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7745 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007746
7747 // First initialization.
7748 // Encoder should be initialized. Next frame should be key frame.
7749 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7750 sink_.SetNumExpectedLayers(number_layers);
7751 int64_t timestamp_ms = kFrameIntervalMs;
7752 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7753 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007754 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007755 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7756 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7757 VideoFrameType::kVideoFrameKey,
7758 VideoFrameType::kVideoFrameKey}));
7759
7760 // Disable top layer.
7761 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7762 config.simulcast_layers[number_layers - 1].active = false;
7763 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7764 sink_.SetNumExpectedLayers(number_layers - 1);
7765 timestamp_ms += kFrameIntervalMs;
7766 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7767 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007768 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007769 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7770 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7771 VideoFrameType::kVideoFrameDelta,
7772 VideoFrameType::kVideoFrameDelta}));
7773
7774 // Re-enable top layer.
7775 // Encoder should be re-initialized. Next frame should be key frame.
7776 config.simulcast_layers[number_layers - 1].active = true;
7777 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7778 sink_.SetNumExpectedLayers(number_layers);
7779 timestamp_ms += kFrameIntervalMs;
7780 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7781 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007782 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007783 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7784 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7785 VideoFrameType::kVideoFrameKey,
7786 VideoFrameType::kVideoFrameKey}));
7787
7788 // Top layer max rate change.
7789 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7790 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7791 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7792 sink_.SetNumExpectedLayers(number_layers);
7793 timestamp_ms += kFrameIntervalMs;
7794 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7795 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007796 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007797 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7798 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7799 VideoFrameType::kVideoFrameDelta,
7800 VideoFrameType::kVideoFrameDelta}));
7801
7802 // Top layer resolution change.
7803 // Encoder should be re-initialized. Next frame should be key frame.
7804 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7805 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7806 sink_.SetNumExpectedLayers(number_layers);
7807 timestamp_ms += kFrameIntervalMs;
7808 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7809 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007810 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007811 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7812 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7813 VideoFrameType::kVideoFrameKey,
7814 VideoFrameType::kVideoFrameKey}));
7815 video_stream_encoder_->Stop();
7816}
7817
Henrik Boström1124ed12021-02-25 10:30:39 +01007818TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7819 const int kFrameWidth = 1280;
7820 const int kFrameHeight = 720;
7821
7822 SetUp();
7823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007825
7826 // Capturing a frame should reconfigure the encoder and expose the encoder
7827 // resolution, which is the same as the input frame.
7828 int64_t timestamp_ms = kFrameIntervalMs;
7829 video_source_.IncomingCapturedFrame(
7830 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7831 WaitForEncodedFrame(timestamp_ms);
7832 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7833 EXPECT_THAT(video_source_.sink_wants().resolutions,
7834 ::testing::ElementsAreArray(
7835 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7836
7837 video_stream_encoder_->Stop();
7838}
7839
7840TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7841 // Pick downscale factors such that we never encode at full resolution - this
7842 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007843 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007844 // encoder should not ask for the frame resolution. This allows video frames
7845 // to have the appearence of one resolution but optimize its internal buffers
7846 // for what is actually encoded.
7847 const size_t kNumSimulcastLayers = 3u;
7848 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7849 const int kFrameWidth = 1280;
7850 const int kFrameHeight = 720;
7851 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7852 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7853 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7854 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7855 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7856 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7857
7858 VideoEncoderConfig config;
7859 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7860 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7861 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7862 config.simulcast_layers[i].active = true;
7863 }
7864 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007865 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007866 "VP8", /*max qp*/ 56, /*screencast*/ false,
7867 /*screenshare enabled*/ false);
7868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007869 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7870 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007871
7872 // Capture a frame with all layers active.
7873 int64_t timestamp_ms = kFrameIntervalMs;
7874 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7875 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7876 video_source_.IncomingCapturedFrame(
7877 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7878 WaitForEncodedFrame(timestamp_ms);
7879 // Expect encoded resolutions to match the expected simulcast layers.
7880 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7881 EXPECT_THAT(
7882 video_source_.sink_wants().resolutions,
7883 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7884
7885 // Capture a frame with one of the layers inactive.
7886 timestamp_ms += kFrameIntervalMs;
7887 config.simulcast_layers[2].active = false;
7888 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7889 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7890 video_source_.IncomingCapturedFrame(
7891 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7892 WaitForEncodedFrame(timestamp_ms);
7893
7894 // Expect encoded resolutions to match the expected simulcast layers.
7895 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7896 EXPECT_THAT(video_source_.sink_wants().resolutions,
7897 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7898
7899 // Capture a frame with all but one layer turned off.
7900 timestamp_ms += kFrameIntervalMs;
7901 config.simulcast_layers[1].active = false;
7902 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7903 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7904 video_source_.IncomingCapturedFrame(
7905 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7906 WaitForEncodedFrame(timestamp_ms);
7907
7908 // Expect encoded resolutions to match the expected simulcast layers.
7909 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7910 EXPECT_THAT(video_source_.sink_wants().resolutions,
7911 ::testing::ElementsAreArray({kLayer0Size}));
7912
7913 video_stream_encoder_->Stop();
7914}
7915
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007916TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007917 ResetEncoder("VP8", 1, 1, 1, false);
7918
Niels Möller8b692902021-06-14 12:04:57 +02007919 // Force encoder reconfig.
7920 video_source_.IncomingCapturedFrame(
7921 CreateFrame(1, codec_width_, codec_height_));
7922 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7923
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007924 // Set QP on encoded frame and pass the frame to encode complete callback.
7925 // Since QP is present QP parsing won't be triggered and the original value
7926 // should be kept.
7927 EncodedImage encoded_image;
7928 encoded_image.qp_ = 123;
7929 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7930 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7931 CodecSpecificInfo codec_info;
7932 codec_info.codecType = kVideoCodecVP8;
7933 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7934 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7935 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7936 video_stream_encoder_->Stop();
7937}
7938
7939TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007940 ResetEncoder("VP8", 1, 1, 1, false);
7941
Niels Möller8b692902021-06-14 12:04:57 +02007942 // Force encoder reconfig.
7943 video_source_.IncomingCapturedFrame(
7944 CreateFrame(1, codec_width_, codec_height_));
7945 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7946
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007947 // Pass an encoded frame without QP to encode complete callback. QP should be
7948 // parsed and set.
7949 EncodedImage encoded_image;
7950 encoded_image.qp_ = -1;
7951 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7952 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7953 CodecSpecificInfo codec_info;
7954 codec_info.codecType = kVideoCodecVP8;
7955 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7956 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7957 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7958 video_stream_encoder_->Stop();
7959}
7960
7961TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7962 webrtc::test::ScopedFieldTrials field_trials(
7963 "WebRTC-QpParsingKillSwitch/Enabled/");
7964
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007965 ResetEncoder("VP8", 1, 1, 1, false);
7966
Niels Möller8b692902021-06-14 12:04:57 +02007967 // Force encoder reconfig.
7968 video_source_.IncomingCapturedFrame(
7969 CreateFrame(1, codec_width_, codec_height_));
7970 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7971
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007972 EncodedImage encoded_image;
7973 encoded_image.qp_ = -1;
7974 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7975 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7976 CodecSpecificInfo codec_info;
7977 codec_info.codecType = kVideoCodecVP8;
7978 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7979 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7980 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7981 video_stream_encoder_->Stop();
7982}
7983
Sergey Silkind19e3b92021-03-16 10:05:30 +00007984TEST_F(VideoStreamEncoderTest,
7985 QualityScalingNotAllowed_QualityScalingDisabled) {
7986 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7987
7988 // Disable scaling settings in encoder info.
7989 fake_encoder_.SetQualityScaling(false);
7990 // Disable quality scaling in encoder config.
7991 video_encoder_config.is_quality_scaling_allowed = false;
7992 ConfigureEncoder(std::move(video_encoder_config));
7993
7994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007996
7997 test::FrameForwarder source;
7998 video_stream_encoder_->SetSource(
7999 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8000 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8002
8003 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8004 WaitForEncodedFrame(1);
8005 video_stream_encoder_->TriggerQualityLow();
8006 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8007
8008 video_stream_encoder_->Stop();
8009}
8010
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008011TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8012 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8013
8014 // Disable scaling settings in encoder info.
8015 fake_encoder_.SetQualityScaling(false);
8016 // Set QP trusted in encoder info.
8017 fake_encoder_.SetIsQpTrusted(true);
8018 // Enable quality scaling in encoder config.
8019 video_encoder_config.is_quality_scaling_allowed = false;
8020 ConfigureEncoder(std::move(video_encoder_config));
8021
8022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008023 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008024
8025 test::FrameForwarder source;
8026 video_stream_encoder_->SetSource(
8027 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8028 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8029 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8030
8031 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8032 WaitForEncodedFrame(1);
8033 video_stream_encoder_->TriggerQualityLow();
8034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8035
8036 video_stream_encoder_->Stop();
8037}
8038
Shuhai Pengf2707702021-09-29 17:19:44 +08008039TEST_F(VideoStreamEncoderTest,
8040 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8041 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8042
8043 // Disable scaling settings in encoder info.
8044 fake_encoder_.SetQualityScaling(false);
8045 // Set QP trusted in encoder info.
8046 fake_encoder_.SetIsQpTrusted(true);
8047 // Enable quality scaling in encoder config.
8048 video_encoder_config.is_quality_scaling_allowed = false;
8049 ConfigureEncoder(std::move(video_encoder_config));
8050
8051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008052 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008053
8054 test::FrameForwarder source;
8055 video_stream_encoder_->SetSource(
8056 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8057 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8058 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8059
8060 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8061 WaitForEncodedFrame(1);
8062 video_stream_encoder_->TriggerQualityLow();
8063 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8064
8065 video_stream_encoder_->Stop();
8066}
8067
8068TEST_F(VideoStreamEncoderTest,
8069 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8070 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8071
8072 // Disable scaling settings in encoder info.
8073 fake_encoder_.SetQualityScaling(false);
8074 // Set QP trusted in encoder info.
8075 fake_encoder_.SetIsQpTrusted(false);
8076 // Enable quality scaling in encoder config.
8077 video_encoder_config.is_quality_scaling_allowed = false;
8078 ConfigureEncoder(std::move(video_encoder_config));
8079
8080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008081 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008082
8083 test::FrameForwarder source;
8084 video_stream_encoder_->SetSource(
8085 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8086 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8087 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8088
8089 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8090 WaitForEncodedFrame(1);
8091 video_stream_encoder_->TriggerQualityLow();
8092 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8093
8094 video_stream_encoder_->Stop();
8095}
8096
8097TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8098 // Set QP trusted in encoder info.
8099 fake_encoder_.SetIsQpTrusted(false);
8100
8101 const int MinEncBitrateKbps = 30;
8102 const int MaxEncBitrateKbps = 100;
8103 const int MinStartBitrateKbp = 50;
8104 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8105 /*frame_size_pixels=*/codec_width_ * codec_height_,
8106 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8107 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8108 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8109
8110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008112
8113 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8114
8115 VideoEncoderConfig video_encoder_config;
8116 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8117 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8118 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8119 MinEncBitrateKbps * 1000;
8120 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8121 kMaxPayloadLength);
8122
8123 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8124 WaitForEncodedFrame(1);
8125 EXPECT_EQ(
8126 MaxEncBitrateKbps,
8127 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8128 EXPECT_EQ(
8129 MinEncBitrateKbps,
8130 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8131
8132 video_stream_encoder_->Stop();
8133}
8134
8135TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8136 // Set QP trusted in encoder info.
8137 fake_encoder_.SetIsQpTrusted(false);
8138
8139 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8140 EncoderInfoSettings::
8141 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8142 codec_width_ * codec_height_,
8143 EncoderInfoSettings::
8144 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8145 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8146
8147 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8148 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8149 const int TargetEncBitrate = MaxEncBitrate;
8150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8151 DataRate::BitsPerSec(TargetEncBitrate),
8152 DataRate::BitsPerSec(TargetEncBitrate),
8153 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8154
8155 VideoEncoderConfig video_encoder_config;
8156 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8157 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8158 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8159 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8160 kMaxPayloadLength);
8161
8162 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8163 WaitForEncodedFrame(1);
8164 EXPECT_EQ(
8165 MaxEncBitrate / 1000,
8166 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8167 EXPECT_EQ(
8168 MinEncBitrate / 1000,
8169 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8170
8171 video_stream_encoder_->Stop();
8172}
8173
Sergey Silkind19e3b92021-03-16 10:05:30 +00008174#if !defined(WEBRTC_IOS)
8175// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8176// disabled by default on iOS.
8177TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8178 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8179
8180 // Disable scaling settings in encoder info.
8181 fake_encoder_.SetQualityScaling(false);
8182 // Enable quality scaling in encoder config.
8183 video_encoder_config.is_quality_scaling_allowed = true;
8184 ConfigureEncoder(std::move(video_encoder_config));
8185
8186 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008187 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008188
8189 test::FrameForwarder source;
8190 video_stream_encoder_->SetSource(
8191 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8192 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8194
8195 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8196 WaitForEncodedFrame(1);
8197 video_stream_encoder_->TriggerQualityLow();
8198 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8199
8200 video_stream_encoder_->Stop();
8201}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008202
8203TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8204 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8205
8206 // Disable scaling settings in encoder info.
8207 fake_encoder_.SetQualityScaling(false);
8208 // Set QP trusted in encoder info.
8209 fake_encoder_.SetIsQpTrusted(true);
8210 // Enable quality scaling in encoder config.
8211 video_encoder_config.is_quality_scaling_allowed = true;
8212 ConfigureEncoder(std::move(video_encoder_config));
8213
8214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008215 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008216
8217 test::FrameForwarder source;
8218 video_stream_encoder_->SetSource(
8219 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8220 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8221 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8222
8223 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8224 WaitForEncodedFrame(1);
8225 video_stream_encoder_->TriggerQualityLow();
8226 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8227
8228 video_stream_encoder_->Stop();
8229}
Shuhai Pengf2707702021-09-29 17:19:44 +08008230
8231TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8232 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8233
8234 // Disable scaling settings in encoder info.
8235 fake_encoder_.SetQualityScaling(false);
8236 // Set QP not trusted in encoder info.
8237 fake_encoder_.SetIsQpTrusted(false);
8238 // Enable quality scaling in encoder config.
8239 video_encoder_config.is_quality_scaling_allowed = true;
8240 ConfigureEncoder(std::move(video_encoder_config));
8241
8242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008243 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008244
8245 test::FrameForwarder source;
8246 video_stream_encoder_->SetSource(
8247 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8248 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8249 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8250
8251 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8252 WaitForEncodedFrame(1);
8253 video_stream_encoder_->TriggerQualityLow();
8254 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8255 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8256 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8257
8258 video_stream_encoder_->Stop();
8259}
8260
8261TEST_F(VideoStreamEncoderTest,
8262 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8263 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8264
8265 // Disable scaling settings in encoder info.
8266 fake_encoder_.SetQualityScaling(false);
8267 // Set QP trusted in encoder info.
8268 fake_encoder_.SetIsQpTrusted(true);
8269 // Enable quality scaling in encoder config.
8270 video_encoder_config.is_quality_scaling_allowed = true;
8271 ConfigureEncoder(std::move(video_encoder_config));
8272
8273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008274 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008275
8276 test::FrameForwarder source;
8277 video_stream_encoder_->SetSource(
8278 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8279 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8280 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8281
8282 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8283 WaitForEncodedFrame(1);
8284 video_stream_encoder_->TriggerQualityLow();
8285 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8287
8288 video_stream_encoder_->Stop();
8289}
8290
8291TEST_F(VideoStreamEncoderTest,
8292 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8293 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8294
8295 // Disable scaling settings in encoder info.
8296 fake_encoder_.SetQualityScaling(false);
8297 // Set QP trusted in encoder info.
8298 fake_encoder_.SetIsQpTrusted(false);
8299 // Enable quality scaling in encoder config.
8300 video_encoder_config.is_quality_scaling_allowed = true;
8301 ConfigureEncoder(std::move(video_encoder_config));
8302
8303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008304 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008305
8306 test::FrameForwarder source;
8307 video_stream_encoder_->SetSource(
8308 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8309 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8310 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8311
8312 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8313 WaitForEncodedFrame(1);
8314 video_stream_encoder_->TriggerQualityLow();
8315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8316
8317 video_stream_encoder_->Stop();
8318}
8319
Sergey Silkind19e3b92021-03-16 10:05:30 +00008320#endif
8321
Henrik Boström56db9ff2021-03-24 09:06:45 +01008322// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8323class VideoStreamEncoderWithRealEncoderTest
8324 : public VideoStreamEncoderTest,
8325 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8326 public:
8327 VideoStreamEncoderWithRealEncoderTest()
8328 : VideoStreamEncoderTest(),
8329 codec_type_(std::get<0>(GetParam())),
8330 allow_i420_conversion_(std::get<1>(GetParam())) {}
8331
8332 void SetUp() override {
8333 VideoStreamEncoderTest::SetUp();
8334 std::unique_ptr<VideoEncoder> encoder;
8335 switch (codec_type_) {
8336 case kVideoCodecVP8:
8337 encoder = VP8Encoder::Create();
8338 break;
8339 case kVideoCodecVP9:
8340 encoder = VP9Encoder::Create();
8341 break;
8342 case kVideoCodecAV1:
8343 encoder = CreateLibaomAv1Encoder();
8344 break;
8345 case kVideoCodecH264:
8346 encoder =
8347 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8348 break;
8349 case kVideoCodecMultiplex:
8350 mock_encoder_factory_for_multiplex_ =
8351 std::make_unique<MockVideoEncoderFactory>();
8352 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8353 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8354 .WillRepeatedly([] { return VP8Encoder::Create(); });
8355 encoder = std::make_unique<MultiplexEncoderAdapter>(
8356 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8357 false);
8358 break;
8359 default:
Artem Titovd3251962021-11-15 16:57:07 +01008360 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008361 }
8362 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8363 }
8364
8365 void TearDown() override {
8366 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008367 // Ensure `video_stream_encoder_` is destroyed before
8368 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008369 video_stream_encoder_.reset();
8370 VideoStreamEncoderTest::TearDown();
8371 }
8372
8373 protected:
8374 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8375 std::unique_ptr<VideoEncoder> encoder) {
8376 // Configure VSE to use the encoder.
8377 encoder_ = std::move(encoder);
8378 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8379 encoder_.get(), &encoder_selector_);
8380 video_send_config_.encoder_settings.encoder_factory =
8381 encoder_proxy_factory_.get();
8382 VideoEncoderConfig video_encoder_config;
8383 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8384 video_encoder_config_ = video_encoder_config.Copy();
8385 ConfigureEncoder(video_encoder_config_.Copy());
8386
8387 // Set bitrate to ensure frame is not dropped.
8388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008389 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008390 }
8391
8392 const VideoCodecType codec_type_;
8393 const bool allow_i420_conversion_;
8394 NiceMock<MockEncoderSelector> encoder_selector_;
8395 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8396 std::unique_ptr<VideoEncoder> encoder_;
8397 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8398};
8399
8400TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8401 auto native_i420_frame = test::CreateMappableNativeFrame(
8402 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8403 video_source_.IncomingCapturedFrame(native_i420_frame);
8404 WaitForEncodedFrame(codec_width_, codec_height_);
8405
8406 auto mappable_native_buffer =
8407 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8408 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8409 mappable_native_buffer->GetMappedFramedBuffers();
8410 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8411 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8412 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8413 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8414}
8415
8416TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8417 auto native_nv12_frame = test::CreateMappableNativeFrame(
8418 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8419 video_source_.IncomingCapturedFrame(native_nv12_frame);
8420 WaitForEncodedFrame(codec_width_, codec_height_);
8421
8422 auto mappable_native_buffer =
8423 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8424 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8425 mappable_native_buffer->GetMappedFramedBuffers();
8426 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8427 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8428 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8429 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8430
8431 if (!allow_i420_conversion_) {
8432 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8433 }
8434}
8435
Erik Språng7444b192021-06-02 14:02:13 +02008436TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8437 if (codec_type_ == kVideoCodecMultiplex) {
8438 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8439 return;
8440 }
8441
8442 const size_t kNumSpatialLayers = 3u;
8443 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8444 const int kFrameWidth = 1280;
8445 const int kFrameHeight = 720;
8446 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8447 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8448 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8449 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8450 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8451 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8452
8453 VideoEncoderConfig config;
8454 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8455 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008456 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008457 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8458 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8459 vp9_settings.numberOfTemporalLayers = 3;
8460 vp9_settings.automaticResizeOn = false;
8461 config.encoder_specific_settings =
8462 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8463 vp9_settings);
8464 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8465 /*fps=*/30.0,
8466 /*first_active_layer=*/0,
8467 /*num_spatial_layers=*/3,
8468 /*num_temporal_layers=*/3,
8469 /*is_screenshare=*/false);
8470 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8471 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008472 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008473 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8474 /*fps=*/30.0,
8475 /*first_active_layer=*/0,
8476 /*num_spatial_layers=*/3,
8477 /*num_temporal_layers=*/3,
8478 /*is_screenshare=*/false);
8479 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8480 } else {
8481 // Simulcast for VP8/H264.
8482 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8483 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8484 config.simulcast_layers[i].scale_resolution_down_by =
8485 kDownscaleFactors[i];
8486 config.simulcast_layers[i].active = true;
8487 }
8488 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8489 // Turn off frame dropping to prevent flakiness.
8490 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8491 h264_settings.frameDroppingOn = false;
8492 config.encoder_specific_settings = rtc::make_ref_counted<
8493 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8494 }
8495 }
8496
8497 auto set_layer_active = [&](int layer_idx, bool active) {
8498 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8499 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8500 config.spatial_layers[layer_idx].active = active;
8501 } else {
8502 config.simulcast_layers[layer_idx].active = active;
8503 }
8504 };
8505
8506 config.video_stream_factory =
8507 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8508 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8509 /*screencast*/ false,
8510 /*screenshare enabled*/ false);
8511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008512 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8513 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008514
8515 // Capture a frame with all layers active.
8516 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8517 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8518 int64_t timestamp_ms = kFrameIntervalMs;
8519 video_source_.IncomingCapturedFrame(
8520 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8521
8522 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8523 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8524
8525 // Capture a frame with one of the layers inactive.
8526 set_layer_active(2, false);
8527 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8528 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8529 timestamp_ms += kFrameIntervalMs;
8530 video_source_.IncomingCapturedFrame(
8531 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8532 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8533
8534 // New target bitrates signaled based on lower resolution.
8535 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8537 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8538 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8539
8540 // Re-enable the top layer.
8541 set_layer_active(2, true);
8542 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8543 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8544 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8545
8546 // Bitrate target adjusted back up to enable HD layer...
8547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8548 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8549 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8550 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8551
8552 // ...then add a new frame.
8553 timestamp_ms += kFrameIntervalMs;
8554 video_source_.IncomingCapturedFrame(
8555 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8556 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8557 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8558
8559 video_stream_encoder_->Stop();
8560}
8561
Henrik Boström56db9ff2021-03-24 09:06:45 +01008562std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8563 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8564 VideoCodecType codec_type = std::get<0>(info.param);
8565 bool allow_i420_conversion = std::get<1>(info.param);
8566 std::string str;
8567 switch (codec_type) {
8568 case kVideoCodecGeneric:
8569 str = "Generic";
8570 break;
8571 case kVideoCodecVP8:
8572 str = "VP8";
8573 break;
8574 case kVideoCodecVP9:
8575 str = "VP9";
8576 break;
8577 case kVideoCodecAV1:
8578 str = "AV1";
8579 break;
8580 case kVideoCodecH264:
8581 str = "H264";
8582 break;
8583 case kVideoCodecMultiplex:
8584 str = "Multiplex";
8585 break;
8586 default:
Artem Titovd3251962021-11-15 16:57:07 +01008587 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008588 }
8589 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8590 return str;
8591}
8592
8593constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8594 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8595constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8596 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8597constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8598 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8599constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8600 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8601#if defined(WEBRTC_USE_H264)
8602constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8603 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8604
8605// The windows compiler does not tolerate #if statements inside the
8606// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8607// and without H264).
8608INSTANTIATE_TEST_SUITE_P(
8609 All,
8610 VideoStreamEncoderWithRealEncoderTest,
8611 ::testing::Values(kVP8DisallowConversion,
8612 kVP9DisallowConversion,
8613 kAV1AllowConversion,
8614 kMultiplexDisallowConversion,
8615 kH264AllowConversion),
8616 TestParametersVideoCodecAndAllowI420ConversionToString);
8617#else
8618INSTANTIATE_TEST_SUITE_P(
8619 All,
8620 VideoStreamEncoderWithRealEncoderTest,
8621 ::testing::Values(kVP8DisallowConversion,
8622 kVP9DisallowConversion,
8623 kAV1AllowConversion,
8624 kMultiplexDisallowConversion),
8625 TestParametersVideoCodecAndAllowI420ConversionToString);
8626#endif
8627
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008628class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8629 protected:
8630 void RunTest(const std::vector<VideoStream>& configs,
8631 const int expected_num_init_encode) {
8632 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008633 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008634 InsertFrameAndWaitForEncoded();
8635 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8636 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008637 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8638 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008639
8640 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8641 ConfigureEncoder(configs[1]);
8642 InsertFrameAndWaitForEncoded();
8643 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8644 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008645 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008646 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008647 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008648
8649 video_stream_encoder_->Stop();
8650 }
8651
8652 void ConfigureEncoder(const VideoStream& stream) {
8653 VideoEncoderConfig config;
8654 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8655 config.max_bitrate_bps = stream.max_bitrate_bps;
8656 config.simulcast_layers[0] = stream;
8657 config.video_stream_factory =
8658 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8659 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8660 /*conference_mode=*/false);
8661 video_stream_encoder_->ConfigureEncoder(std::move(config),
8662 kMaxPayloadLength);
8663 }
8664
8665 void OnBitrateUpdated(DataRate bitrate) {
8666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8667 bitrate, bitrate, bitrate, 0, 0, 0);
8668 }
8669
8670 void InsertFrameAndWaitForEncoded() {
8671 timestamp_ms_ += kFrameIntervalMs;
8672 video_source_.IncomingCapturedFrame(
8673 CreateFrame(timestamp_ms_, kWidth, kHeight));
8674 sink_.WaitForEncodedFrame(timestamp_ms_);
8675 }
8676
8677 void ExpectEqual(const VideoCodec& actual,
8678 const VideoStream& expected) const {
8679 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8680 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8681 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8682 static_cast<unsigned int>(expected.min_bitrate_bps));
8683 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8684 static_cast<unsigned int>(expected.max_bitrate_bps));
8685 EXPECT_EQ(actual.simulcastStream[0].width,
8686 kWidth / expected.scale_resolution_down_by);
8687 EXPECT_EQ(actual.simulcastStream[0].height,
8688 kHeight / expected.scale_resolution_down_by);
8689 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8690 expected.num_temporal_layers);
8691 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8692 }
8693
8694 VideoStream DefaultConfig() const {
8695 VideoStream stream;
8696 stream.max_framerate = 25;
8697 stream.min_bitrate_bps = 35000;
8698 stream.max_bitrate_bps = 900000;
8699 stream.scale_resolution_down_by = 1.0;
8700 stream.num_temporal_layers = 1;
8701 stream.bitrate_priority = 1.0;
8702 stream.scalability_mode = "";
8703 return stream;
8704 }
8705
8706 const int kWidth = 640;
8707 const int kHeight = 360;
8708 int64_t timestamp_ms_ = 0;
8709};
8710
8711TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8712 VideoStream config1 = DefaultConfig();
8713 VideoStream config2 = config1;
8714 config2.max_framerate++;
8715
8716 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8717}
8718
8719TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8720 VideoStream config1 = DefaultConfig();
8721 VideoStream config2 = config1;
8722 config2.min_bitrate_bps += 10000;
8723
8724 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8725}
8726
8727TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8728 VideoStream config1 = DefaultConfig();
8729 VideoStream config2 = config1;
8730 config2.max_bitrate_bps += 100000;
8731
8732 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8733}
8734
8735TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8736 VideoStream config1 = DefaultConfig();
8737 VideoStream config2 = config1;
8738 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8739
8740 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8741}
8742
8743TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8744 VideoStream config1 = DefaultConfig();
8745 VideoStream config2 = config1;
8746 config2.scale_resolution_down_by *= 2;
8747
8748 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8749}
8750
8751TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8752 VideoStream config1 = DefaultConfig();
8753 VideoStream config2 = config1;
8754 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8755
8756 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8757}
8758
8759TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8760 VideoStream config1 = DefaultConfig();
8761 VideoStream config2 = config1;
8762 config2.scalability_mode = "L1T2";
8763
8764 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8765}
8766
Markus Handellb4e96d42021-11-05 12:00:55 +01008767TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8768 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8769 auto* adapter_ptr = adapter.get();
8770 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01008771 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8772 nullptr;
8773 EXPECT_CALL(*adapter_ptr, Initialize)
8774 .WillOnce(Invoke([&video_stream_encoder_callback](
8775 FrameCadenceAdapterInterface::Callback* callback) {
8776 video_stream_encoder_callback = callback;
8777 }));
8778 TaskQueueBase* encoder_queue = nullptr;
8779 auto video_stream_encoder =
8780 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01008781
Markus Handelle59fee82021-12-23 09:29:23 +01008782 // First a call before we know the frame size and hence cannot compute the
8783 // number of simulcast layers.
8784 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8785 &FrameCadenceAdapterInterface::
8786 ZeroHertzModeParams::num_simulcast_layers,
8787 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01008788 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01008789 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01008790 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8791 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01008792 factory.DepleteTaskQueues();
8793
8794 // Then a call as we've computed the number of simulcast layers after a passed
8795 // frame.
8796 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8797 &FrameCadenceAdapterInterface::
8798 ZeroHertzModeParams::num_simulcast_layers,
8799 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01008800 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01008801 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008802 Mock::VerifyAndClearExpectations(adapter_ptr);
8803
Markus Handelle59fee82021-12-23 09:29:23 +01008804 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01008805 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01008806 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01008807 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01008808 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8809 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008810 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01008811 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008812}
8813
8814TEST(VideoStreamEncoderFrameCadenceTest,
8815 ForwardsFramesIntoFrameCadenceAdapter) {
8816 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8817 auto* adapter_ptr = adapter.get();
8818 test::FrameForwarder video_source;
8819 SimpleVideoStreamEncoderFactory factory;
8820 auto video_stream_encoder = factory.Create(std::move(adapter));
8821 video_stream_encoder->SetSource(
8822 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8823
8824 EXPECT_CALL(*adapter_ptr, OnFrame);
8825 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8826 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01008827 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01008828}
8829
Markus Handellee225432021-11-29 12:35:12 +01008830TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8831 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8832 auto* adapter_ptr = adapter.get();
8833 test::FrameForwarder video_source;
8834 SimpleVideoStreamEncoderFactory factory;
8835 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8836 nullptr;
8837 EXPECT_CALL(*adapter_ptr, Initialize)
8838 .WillOnce(Invoke([&video_stream_encoder_callback](
8839 FrameCadenceAdapterInterface::Callback* callback) {
8840 video_stream_encoder_callback = callback;
8841 }));
8842 TaskQueueBase* encoder_queue = nullptr;
8843 auto video_stream_encoder =
8844 factory.Create(std::move(adapter), &encoder_queue);
8845
8846 // This is just to make the VSE operational. We'll feed a frame directly by
8847 // the callback interface.
8848 video_stream_encoder->SetSource(
8849 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8850
8851 VideoEncoderConfig video_encoder_config;
8852 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
8853 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
8854 /*max_data_payload_length=*/1000);
8855
8856 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
8857 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01008858 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01008859 factory.DepleteTaskQueues();
8860}
8861
Markus Handell8d87c462021-12-16 11:37:16 +01008862TEST(VideoStreamEncoderFrameCadenceTest,
8863 DeactivatesActivatesLayersOnBitrateChanges) {
8864 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8865 auto* adapter_ptr = adapter.get();
8866 SimpleVideoStreamEncoderFactory factory;
8867 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8868 nullptr;
8869 EXPECT_CALL(*adapter_ptr, Initialize)
8870 .WillOnce(Invoke([&video_stream_encoder_callback](
8871 FrameCadenceAdapterInterface::Callback* callback) {
8872 video_stream_encoder_callback = callback;
8873 }));
8874 TaskQueueBase* encoder_queue = nullptr;
8875 auto video_stream_encoder =
8876 factory.Create(std::move(adapter), &encoder_queue);
8877
8878 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
8879 // {150000, 450000}.
8880 VideoEncoderConfig video_encoder_config;
8881 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8882 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8883 kMaxPayloadLength);
8884 // Ensure an encoder is created.
8885 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8886
8887 // Both layers enabled at 1 MBit/s.
8888 video_stream_encoder->OnBitrateUpdated(
8889 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8890 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8891 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8892 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8893 factory.DepleteTaskQueues();
8894 Mock::VerifyAndClearExpectations(adapter_ptr);
8895
8896 // Layer 1 disabled at 200 KBit/s.
8897 video_stream_encoder->OnBitrateUpdated(
8898 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
8899 DataRate::KilobitsPerSec(200), 0, 0, 0);
8900 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8901 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8902 factory.DepleteTaskQueues();
8903 Mock::VerifyAndClearExpectations(adapter_ptr);
8904
8905 // All layers off at suspended video.
8906 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
8907 DataRate::Zero(), 0, 0, 0);
8908 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
8909 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8910 factory.DepleteTaskQueues();
8911 Mock::VerifyAndClearExpectations(adapter_ptr);
8912
8913 // Both layers enabled again back at 1 MBit/s.
8914 video_stream_encoder->OnBitrateUpdated(
8915 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8916 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8917 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8918 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8919 factory.DepleteTaskQueues();
8920}
8921
8922TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
8923 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8924 auto* adapter_ptr = adapter.get();
8925 SimpleVideoStreamEncoderFactory factory;
8926 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8927 nullptr;
8928 EXPECT_CALL(*adapter_ptr, Initialize)
8929 .WillOnce(Invoke([&video_stream_encoder_callback](
8930 FrameCadenceAdapterInterface::Callback* callback) {
8931 video_stream_encoder_callback = callback;
8932 }));
8933 TaskQueueBase* encoder_queue = nullptr;
8934 auto video_stream_encoder =
8935 factory.Create(std::move(adapter), &encoder_queue);
8936
8937 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
8938 VideoEncoderConfig video_encoder_config;
8939 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8940 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8941 kMaxPayloadLength);
8942 video_stream_encoder->OnBitrateUpdated(
8943 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8944 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8945
8946 // Pass a frame which has unconverged results.
8947 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8948 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8949 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8950 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8951 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
8952 CodecSpecificInfo codec_specific;
8953 codec_specific.codecType = kVideoCodecGeneric;
8954 return codec_specific;
8955 }));
8956 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
8957 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8958 factory.DepleteTaskQueues();
8959 Mock::VerifyAndClearExpectations(adapter_ptr);
8960 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8961
8962 // Pass a frame which converges in layer 0 and not in layer 1.
8963 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
8964 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8965 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8966 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8967 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
8968 CodecSpecificInfo codec_specific;
8969 codec_specific.codecType = kVideoCodecGeneric;
8970 return codec_specific;
8971 }));
8972 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
8973 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8974 factory.DepleteTaskQueues();
8975 Mock::VerifyAndClearExpectations(adapter_ptr);
8976 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8977}
8978
Markus Handell2e0f4f02021-12-21 19:14:58 +01008979TEST(VideoStreamEncoderFrameCadenceTest,
8980 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
8981 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8982 auto* adapter_ptr = adapter.get();
8983 MockVideoSourceInterface mock_source;
8984 SimpleVideoStreamEncoderFactory factory;
8985 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8986 nullptr;
8987 EXPECT_CALL(*adapter_ptr, Initialize)
8988 .WillOnce(Invoke([&video_stream_encoder_callback](
8989 FrameCadenceAdapterInterface::Callback* callback) {
8990 video_stream_encoder_callback = callback;
8991 }));
8992 TaskQueueBase* encoder_queue = nullptr;
8993 auto video_stream_encoder =
8994 factory.Create(std::move(adapter), &encoder_queue);
8995 video_stream_encoder->SetSource(
8996 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8997 VideoEncoderConfig config;
8998 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8999 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9000 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9001 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9002 // Ensure the encoder is set up.
9003 factory.DepleteTaskQueues();
9004
9005 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest).WillOnce(Return(true));
9006 EXPECT_CALL(mock_source, RequestRefreshFrame);
9007 video_stream_encoder->SendKeyFrame();
9008 factory.DepleteTaskQueues();
9009 Mock::VerifyAndClearExpectations(adapter_ptr);
9010 Mock::VerifyAndClearExpectations(&mock_source);
9011
9012 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest).WillOnce(Return(false));
9013 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9014 video_stream_encoder->SendKeyFrame();
9015 factory.DepleteTaskQueues();
9016}
9017
perkj26091b12016-09-01 01:17:40 -07009018} // namespace webrtc