blob: 3d6956ebf7b594168fb3009a1d4639dabc6e9cf5 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010022#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020023#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010024#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010025#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010026#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080027#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020029#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020030#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010031#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010032#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020033#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020034#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010035#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020036#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010037#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020038#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070039#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080040#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020041#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010042#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
43#include "modules/video_coding/codecs/h264/include/h264.h"
44#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
45#include "modules/video_coding/codecs/vp8/include/vp8.h"
46#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020047#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020048#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020049#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010050#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020051#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010052#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020053#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080055#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020056#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010057#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020058#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/encoder_settings.h"
60#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020061#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010065#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020066#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020067#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010068#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020069#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070070
71namespace webrtc {
72
sprang57c2fff2017-01-16 06:24:02 -080073using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020074using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020075using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020076using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020077using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020078using ::testing::Ge;
79using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010080using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Le;
82using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010083using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010084using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010085using ::testing::NiceMock;
Markus Handell8d87c462021-12-16 11:37:16 +010086using ::testing::Not;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010089using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020090using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080091
perkj803d97f2016-11-01 11:45:46 -070092namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020093const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010094const int kQpLow = 1;
95const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020096const int kMinFramerateFps = 2;
97const int kMinBalancedFramerateFps = 7;
98const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080099const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200100const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
101const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
102const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
103const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800104const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700105const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200106const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200107const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800112
Asa Persson606d3cb2021-10-04 10:07:11 +0200113uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200114 0x00, 0x00, 0x03, 0x03, 0xF4,
115 0x05, 0x03, 0xC7, 0xE0, 0x1B,
116 0x41, 0x10, 0x8D, 0x00};
117
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100118const uint8_t kCodedFrameVp8Qp25[] = {
119 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
120 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
121 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
122
Markus Handell8d87c462021-12-16 11:37:16 +0100123void PassAFrame(
124 TaskQueueBase* encoder_queue,
125 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
126 int64_t ntp_time_ms) {
127 encoder_queue->PostTask(
128 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
129 video_stream_encoder_callback->OnFrame(
130 Timestamp::Millis(ntp_time_ms), 1,
131 VideoFrame::Builder()
132 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
133 /*width=*/16, /*height=*/16))
134 .build());
135 }));
136}
137
perkj803d97f2016-11-01 11:45:46 -0700138class TestBuffer : public webrtc::I420Buffer {
139 public:
140 TestBuffer(rtc::Event* event, int width, int height)
141 : I420Buffer(width, height), event_(event) {}
142
143 private:
144 friend class rtc::RefCountedObject<TestBuffer>;
145 ~TestBuffer() override {
146 if (event_)
147 event_->Set();
148 }
149 rtc::Event* const event_;
150};
151
Henrik Boström56db9ff2021-03-24 09:06:45 +0100152// A fake native buffer that can't be converted to I420. Upon scaling, it
153// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700154class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
155 public:
156 FakeNativeBuffer(rtc::Event* event, int width, int height)
157 : event_(event), width_(width), height_(height) {}
158 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
159 int width() const override { return width_; }
160 int height() const override { return height_; }
161 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
162 return nullptr;
163 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100164 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
165 int offset_x,
166 int offset_y,
167 int crop_width,
168 int crop_height,
169 int scaled_width,
170 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200171 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
172 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100173 }
Noah Richards51db4212019-06-12 06:59:12 -0700174
175 private:
176 friend class rtc::RefCountedObject<FakeNativeBuffer>;
177 ~FakeNativeBuffer() override {
178 if (event_)
179 event_->Set();
180 }
181 rtc::Event* const event_;
182 const int width_;
183 const int height_;
184};
185
Evan Shrubsole895556e2020-10-05 09:15:13 +0200186// A fake native buffer that is backed by an NV12 buffer.
187class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
188 public:
189 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
190 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
191
192 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
193 int width() const override { return nv12_buffer_->width(); }
194 int height() const override { return nv12_buffer_->height(); }
195 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
196 return nv12_buffer_->ToI420();
197 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200198 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
199 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
200 if (absl::c_find(types, Type::kNV12) != types.end()) {
201 return nv12_buffer_;
202 }
203 return nullptr;
204 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200205 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
206
207 private:
208 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
209 ~FakeNV12NativeBuffer() override {
210 if (event_)
211 event_->Set();
212 }
213 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
214 rtc::Event* const event_;
215};
216
Niels Möller7dc26b72017-12-06 10:27:48 +0100217class CpuOveruseDetectorProxy : public OveruseFrameDetector {
218 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200219 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
220 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200221 last_target_framerate_fps_(-1),
222 framerate_updated_event_(true /* manual_reset */,
223 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100224 virtual ~CpuOveruseDetectorProxy() {}
225
226 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200227 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100228 last_target_framerate_fps_ = framerate_fps;
229 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200230 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 }
232
233 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200234 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100235 return last_target_framerate_fps_;
236 }
237
Niels Möller4db138e2018-04-19 09:04:13 +0200238 CpuOveruseOptions GetOptions() { return options_; }
239
Henrik Boström381d1092020-05-12 18:49:07 +0200240 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
241
Niels Möller7dc26b72017-12-06 10:27:48 +0100242 private:
Markus Handella3765182020-07-08 13:13:32 +0200243 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100244 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200245 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100246};
247
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200248class FakeVideoSourceRestrictionsListener
249 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200250 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200251 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200252 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200254 RTC_DCHECK(was_restrictions_updated_);
255 }
256
257 rtc::Event* restrictions_updated_event() {
258 return &restrictions_updated_event_;
259 }
260
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200261 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200262 void OnVideoSourceRestrictionsUpdated(
263 VideoSourceRestrictions restrictions,
264 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200265 rtc::scoped_refptr<Resource> reason,
266 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200267 was_restrictions_updated_ = true;
268 restrictions_updated_event_.Set();
269 }
270
271 private:
272 bool was_restrictions_updated_;
273 rtc::Event restrictions_updated_event_;
274};
275
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200276auto WantsFps(Matcher<int> fps_matcher) {
277 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
278 fps_matcher);
279}
280
281auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
282 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
283 AllOf(max_pixel_matcher, Gt(0)));
284}
285
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200286auto ResolutionMax() {
287 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200288 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200289 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
290 Eq(absl::nullopt)));
291}
292
293auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200294 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200295}
296
297auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200298 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200299}
300
301auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200302 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200303}
304
305auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200306 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200307}
308
309auto FpsMaxResolutionMax() {
310 return AllOf(FpsMax(), ResolutionMax());
311}
312
313auto UnlimitedSinkWants() {
314 return AllOf(FpsUnlimited(), ResolutionMax());
315}
316
317auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
318 Matcher<int> fps_range_matcher;
319
320 if (last_frame_pixels <= 320 * 240) {
321 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200322 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200323 fps_range_matcher = AllOf(Ge(10), Le(15));
324 } else if (last_frame_pixels <= 640 * 480) {
325 fps_range_matcher = Ge(15);
326 } else {
327 fps_range_matcher = Eq(kDefaultFramerate);
328 }
329 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
330 fps_range_matcher);
331}
332
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200333auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
336}
337
338auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
340}
341
342auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
343 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
344}
345
346auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
347 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
348 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
349}
350
351auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
352 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
353 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
354}
355
356auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
357 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
358 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
359}
360
361auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
362 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
363 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
364}
365
mflodmancc3d4422017-08-03 08:27:51 -0700366class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700367 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100368 VideoStreamEncoderUnderTest(
369 TimeController* time_controller,
370 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
371 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
372 encoder_queue,
373 SendStatisticsProxy* stats_proxy,
374 const VideoStreamEncoderSettings& settings,
375 VideoStreamEncoder::BitrateAllocationCallbackType
376 allocation_callback_type)
377 : VideoStreamEncoder(time_controller->GetClock(),
378 1 /* number_of_cores */,
379 stats_proxy,
380 settings,
381 std::unique_ptr<OveruseFrameDetector>(
382 overuse_detector_proxy_ =
383 new CpuOveruseDetectorProxy(stats_proxy)),
384 std::move(cadence_adapter),
385 std::move(encoder_queue),
386 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200387 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200388 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200389 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200390 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200391 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200392 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200393 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200394 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100395 }
perkj803d97f2016-11-01 11:45:46 -0700396
Henrik Boström381d1092020-05-12 18:49:07 +0200397 void SetSourceAndWaitForRestrictionsUpdated(
398 rtc::VideoSourceInterface<VideoFrame>* source,
399 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 FakeVideoSourceRestrictionsListener listener;
401 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200402 SetSource(source, degradation_preference);
403 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200404 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200405 }
406
407 void SetSourceAndWaitForFramerateUpdated(
408 rtc::VideoSourceInterface<VideoFrame>* source,
409 const DegradationPreference& degradation_preference) {
410 overuse_detector_proxy_->framerate_updated_event()->Reset();
411 SetSource(source, degradation_preference);
412 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
413 }
414
415 void OnBitrateUpdatedAndWaitForManagedResources(
416 DataRate target_bitrate,
417 DataRate stable_target_bitrate,
418 DataRate link_allocation,
419 uint8_t fraction_lost,
420 int64_t round_trip_time_ms,
421 double cwnd_reduce_ratio) {
422 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
423 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
424 // Bitrate is updated on the encoder queue.
425 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200426 }
427
kthelgason2fc52542017-03-03 00:24:41 -0800428 // This is used as a synchronisation mechanism, to make sure that the
429 // encoder queue is not blocked before we start sending it frames.
430 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100431 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800432 }
433
Henrik Boström91aa7322020-04-28 12:24:33 +0200434 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200435 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200436 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200437 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200438 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200439 event.Set();
440 });
441 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100442 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200444
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 void TriggerCpuUnderuse() {
446 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200447 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200448 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 event.Set();
450 });
451 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100452 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200453 }
kthelgason876222f2016-11-29 01:44:11 -0800454
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200456 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200457 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200458 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200459 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200460 event.Set();
461 });
462 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100463 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200464 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200467 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200468 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 event.Set();
470 });
471 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100472 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200473 }
474
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200475 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100476 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200477 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
478 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200479 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700480};
481
Noah Richards51db4212019-06-12 06:59:12 -0700482// Simulates simulcast behavior and makes highest stream resolutions divisible
483// by 4.
484class CroppingVideoStreamFactory
485 : public VideoEncoderConfig::VideoStreamFactoryInterface {
486 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200487 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700488
489 private:
490 std::vector<VideoStream> CreateEncoderStreams(
491 int width,
492 int height,
493 const VideoEncoderConfig& encoder_config) override {
494 std::vector<VideoStream> streams = test::CreateVideoStreams(
495 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700496 return streams;
497 }
Noah Richards51db4212019-06-12 06:59:12 -0700498};
499
sprangb1ca0732017-02-01 08:38:12 -0800500class AdaptingFrameForwarder : public test::FrameForwarder {
501 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200502 explicit AdaptingFrameForwarder(TimeController* time_controller)
503 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700504 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800505
506 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200507 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800508 adaptation_enabled_ = enabled;
509 }
510
asaperssonfab67072017-04-04 05:51:49 -0700511 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200512 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800513 return adaptation_enabled_;
514 }
515
Henrik Boström1124ed12021-02-25 10:30:39 +0100516 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
517 // the resolution or frame rate was different than it is currently. If
518 // something else is modified, such as encoder resolutions, but the resolution
519 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700520 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700522 return last_wants_;
523 }
524
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200525 absl::optional<int> last_sent_width() const { return last_width_; }
526 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800527
sprangb1ca0732017-02-01 08:38:12 -0800528 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200529 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100530 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200531
sprangb1ca0732017-02-01 08:38:12 -0800532 int cropped_width = 0;
533 int cropped_height = 0;
534 int out_width = 0;
535 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700536 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000537 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
538 << "w=" << video_frame.width()
539 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700540 if (adapter_.AdaptFrameResolution(
541 video_frame.width(), video_frame.height(),
542 video_frame.timestamp_us() * 1000, &cropped_width,
543 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100544 VideoFrame adapted_frame =
545 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200546 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100547 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200548 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100549 .set_timestamp_ms(99)
550 .set_rotation(kVideoRotation_0)
551 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100552 if (video_frame.has_update_rect()) {
553 adapted_frame.set_update_rect(
554 video_frame.update_rect().ScaleWithFrame(
555 video_frame.width(), video_frame.height(), 0, 0,
556 video_frame.width(), video_frame.height(), out_width,
557 out_height));
558 }
sprangc5d62e22017-04-02 23:53:04 -0700559 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800560 last_width_.emplace(adapted_frame.width());
561 last_height_.emplace(adapted_frame.height());
562 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200563 last_width_ = absl::nullopt;
564 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700565 }
sprangb1ca0732017-02-01 08:38:12 -0800566 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000567 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800568 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800569 last_width_.emplace(video_frame.width());
570 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800571 }
572 }
573
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200574 void OnOutputFormatRequest(int width, int height) {
575 absl::optional<std::pair<int, int>> target_aspect_ratio =
576 std::make_pair(width, height);
577 absl::optional<int> max_pixel_count = width * height;
578 absl::optional<int> max_fps;
579 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
580 max_fps);
581 }
582
sprangb1ca0732017-02-01 08:38:12 -0800583 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
584 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200585 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100586 rtc::VideoSinkWants prev_wants = sink_wants_locked();
587 bool did_adapt =
588 prev_wants.max_pixel_count != wants.max_pixel_count ||
589 prev_wants.target_pixel_count != wants.target_pixel_count ||
590 prev_wants.max_framerate_fps != wants.max_framerate_fps;
591 if (did_adapt) {
592 last_wants_ = prev_wants;
593 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100594 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200595 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800596 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200597
598 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800599 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200600 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
601 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200602 absl::optional<int> last_width_;
603 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800604};
sprangc5d62e22017-04-02 23:53:04 -0700605
Niels Möller213618e2018-07-24 09:29:58 +0200606// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700607class MockableSendStatisticsProxy : public SendStatisticsProxy {
608 public:
609 MockableSendStatisticsProxy(Clock* clock,
610 const VideoSendStream::Config& config,
611 VideoEncoderConfig::ContentType content_type)
612 : SendStatisticsProxy(clock, config, content_type) {}
613
614 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200615 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700616 if (mock_stats_)
617 return *mock_stats_;
618 return SendStatisticsProxy::GetStats();
619 }
620
Niels Möller213618e2018-07-24 09:29:58 +0200621 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200622 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200623 if (mock_stats_)
624 return mock_stats_->input_frame_rate;
625 return SendStatisticsProxy::GetInputFrameRate();
626 }
sprangc5d62e22017-04-02 23:53:04 -0700627 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200628 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700629 mock_stats_.emplace(stats);
630 }
631
632 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200633 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700634 mock_stats_.reset();
635 }
636
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200637 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
638 on_frame_dropped_ = std::move(callback);
639 }
640
sprangc5d62e22017-04-02 23:53:04 -0700641 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200642 void OnFrameDropped(DropReason reason) override {
643 SendStatisticsProxy::OnFrameDropped(reason);
644 if (on_frame_dropped_)
645 on_frame_dropped_(reason);
646 }
647
Markus Handella3765182020-07-08 13:13:32 +0200648 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200649 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200650 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700651};
652
Markus Handellb4e96d42021-11-05 12:00:55 +0100653class SimpleVideoStreamEncoderFactory {
654 public:
655 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
656 public:
657 using VideoStreamEncoder::VideoStreamEncoder;
658 ~AdaptedVideoStreamEncoder() { Stop(); }
659 };
660
Markus Handell8d87c462021-12-16 11:37:16 +0100661 class MockFakeEncoder : public test::FakeEncoder {
662 public:
663 using FakeEncoder::FakeEncoder;
664 MOCK_METHOD(CodecSpecificInfo,
665 EncodeHook,
666 (EncodedImage & encoded_image,
667 rtc::scoped_refptr<EncodedImageBuffer> buffer),
668 (override));
669 };
670
Markus Handellee225432021-11-29 12:35:12 +0100671 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100672 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100673 encoder_settings_.bitrate_allocator_factory =
674 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100675 }
676
677 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
Markus Handellee225432021-11-29 12:35:12 +0100678 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
679 TaskQueueBase** encoder_queue_ptr = nullptr) {
680 auto encoder_queue =
681 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
682 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
683 if (encoder_queue_ptr)
684 *encoder_queue_ptr = encoder_queue.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100685 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
686 time_controller_.GetClock(),
687 /*number_of_cores=*/1,
688 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
689 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100690 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100691 VideoStreamEncoder::BitrateAllocationCallbackType::
692 kVideoBitrateAllocation);
693 result->SetSink(&sink_, /*rotation_applied=*/false);
694 return result;
695 }
696
Markus Handell9a478b52021-11-18 16:07:01 +0100697 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100698 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100699
Markus Handellb4e96d42021-11-05 12:00:55 +0100700 private:
701 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
702 public:
703 ~NullEncoderSink() override = default;
704 void OnEncoderConfigurationChanged(
705 std::vector<VideoStream> streams,
706 bool is_svc,
707 VideoEncoderConfig::ContentType content_type,
708 int min_transmit_bitrate_bps) override {}
709 void OnBitrateAllocationUpdated(
710 const VideoBitrateAllocation& allocation) override {}
711 void OnVideoLayersAllocationUpdated(
712 VideoLayersAllocation allocation) override {}
713 Result OnEncodedImage(
714 const EncodedImage& encoded_image,
715 const CodecSpecificInfo* codec_specific_info) override {
716 return Result(EncodedImageCallback::Result::OK);
717 }
718 };
719
Markus Handellee225432021-11-29 12:35:12 +0100720 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
721 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
722 time_controller_.CreateTaskQueueFactory()};
723 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
724 std::make_unique<MockableSendStatisticsProxy>(
725 time_controller_.GetClock(),
726 VideoSendStream::Config(nullptr),
727 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
728 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
729 CreateBuiltinVideoBitrateAllocatorFactory();
730 VideoStreamEncoderSettings encoder_settings_{
731 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100732 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
733 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100734 NullEncoderSink sink_;
735};
736
737class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
738 public:
739 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100740 MOCK_METHOD(void,
741 SetZeroHertzModeEnabled,
742 (absl::optional<ZeroHertzModeParams>),
743 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100744 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100745 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
746 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100747 MOCK_METHOD(void,
748 UpdateLayerQualityConvergence,
749 (int spatial_index, bool converged),
750 (override));
751 MOCK_METHOD(void,
752 UpdateLayerStatus,
753 (int spatial_index, bool enabled),
754 (override));
Markus Handell2e0f4f02021-12-21 19:14:58 +0100755 MOCK_METHOD(bool, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100756};
757
philipel9b058032020-02-10 11:30:00 +0100758class MockEncoderSelector
759 : public VideoEncoderFactory::EncoderSelectorInterface {
760 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200761 MOCK_METHOD(void,
762 OnCurrentEncoder,
763 (const SdpVideoFormat& format),
764 (override));
765 MOCK_METHOD(absl::optional<SdpVideoFormat>,
766 OnAvailableBitrate,
767 (const DataRate& rate),
768 (override));
769 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100770};
771
Markus Handell2e0f4f02021-12-21 19:14:58 +0100772class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
773 public:
774 MOCK_METHOD(void,
775 AddOrUpdateSink,
776 (rtc::VideoSinkInterface<VideoFrame>*,
777 const rtc::VideoSinkWants&),
778 (override));
779 MOCK_METHOD(void,
780 RemoveSink,
781 (rtc::VideoSinkInterface<VideoFrame>*),
782 (override));
783 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
784};
785
perkj803d97f2016-11-01 11:45:46 -0700786} // namespace
787
mflodmancc3d4422017-08-03 08:27:51 -0700788class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700789 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200790 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700791
mflodmancc3d4422017-08-03 08:27:51 -0700792 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700793 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700794 codec_width_(320),
795 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200796 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200797 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200798 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700799 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200800 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700801 video_send_config_,
802 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200803 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700804
805 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700806 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700807 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200808 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800809 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200810 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200811 video_send_config_.rtp.payload_name = "FAKE";
812 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700813
Per512ecb32016-09-23 15:52:06 +0200814 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200815 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200816 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
817 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
818 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100819 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700820
Niels Möllerf1338562018-04-26 09:51:47 +0200821 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800822 }
823
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100824 void ConfigureEncoder(
825 VideoEncoderConfig video_encoder_config,
826 VideoStreamEncoder::BitrateAllocationCallbackType
827 allocation_callback_type =
828 VideoStreamEncoder::BitrateAllocationCallbackType::
829 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700830 if (video_stream_encoder_)
831 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100832
833 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
834 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
835 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
836 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
837 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
838 encoder_queue_ptr);
839 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
840 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
841 stats_proxy_.get(), video_send_config_.encoder_settings,
842 allocation_callback_type);
Asa Persson606d3cb2021-10-04 10:07:11 +0200843 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700844 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700845 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200846 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700847 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200848 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700849 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800850 }
851
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100852 void ResetEncoder(const std::string& payload_name,
853 size_t num_streams,
854 size_t num_temporal_layers,
855 unsigned char num_spatial_layers,
856 bool screenshare,
857 VideoStreamEncoder::BitrateAllocationCallbackType
858 allocation_callback_type =
859 VideoStreamEncoder::BitrateAllocationCallbackType::
860 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200861 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800862
863 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200864 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
865 num_streams, &video_encoder_config);
866 for (auto& layer : video_encoder_config.simulcast_layers) {
867 layer.num_temporal_layers = num_temporal_layers;
868 layer.max_framerate = kDefaultFramerate;
869 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100870 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200871 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700872 video_encoder_config.content_type =
873 screenshare ? VideoEncoderConfig::ContentType::kScreen
874 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700875 if (payload_name == "VP9") {
876 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
877 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200878 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700879 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200880 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
881 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700882 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100883 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700884 }
885
sprang57c2fff2017-01-16 06:24:02 -0800886 VideoFrame CreateFrame(int64_t ntp_time_ms,
887 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200888 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200889 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200890 destruction_event, codec_width_, codec_height_))
891 .set_ntp_time_ms(ntp_time_ms)
892 .set_timestamp_ms(99)
893 .set_rotation(kVideoRotation_0)
894 .build();
perkj26091b12016-09-01 01:17:40 -0700895 }
896
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100897 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
898 rtc::Event* destruction_event,
899 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200900 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200901 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200902 destruction_event, codec_width_, codec_height_))
903 .set_ntp_time_ms(ntp_time_ms)
904 .set_timestamp_ms(99)
905 .set_rotation(kVideoRotation_0)
906 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
907 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100908 }
909
sprang57c2fff2017-01-16 06:24:02 -0800910 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200911 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
912 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200913 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200914 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200915 .set_ntp_time_ms(ntp_time_ms)
916 .set_timestamp_ms(ntp_time_ms)
917 .set_rotation(kVideoRotation_0)
918 .build();
perkj803d97f2016-11-01 11:45:46 -0700919 }
920
Evan Shrubsole895556e2020-10-05 09:15:13 +0200921 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200922 return VideoFrame::Builder()
923 .set_video_frame_buffer(NV12Buffer::Create(width, height))
924 .set_ntp_time_ms(ntp_time_ms)
925 .set_timestamp_ms(ntp_time_ms)
926 .set_rotation(kVideoRotation_0)
927 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200928 }
929
Noah Richards51db4212019-06-12 06:59:12 -0700930 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
931 rtc::Event* destruction_event,
932 int width,
933 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200934 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200935 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200936 destruction_event, width, height))
937 .set_ntp_time_ms(ntp_time_ms)
938 .set_timestamp_ms(99)
939 .set_rotation(kVideoRotation_0)
940 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700941 }
942
Evan Shrubsole895556e2020-10-05 09:15:13 +0200943 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
944 rtc::Event* destruction_event,
945 int width,
946 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200947 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200948 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200949 destruction_event, width, height))
950 .set_ntp_time_ms(ntp_time_ms)
951 .set_timestamp_ms(99)
952 .set_rotation(kVideoRotation_0)
953 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200954 }
955
Noah Richards51db4212019-06-12 06:59:12 -0700956 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
957 rtc::Event* destruction_event) const {
958 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
959 codec_height_);
960 }
961
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100962 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200964 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100965
966 video_source_.IncomingCapturedFrame(
967 CreateFrame(1, codec_width_, codec_height_));
968 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200969 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100970 }
971
sprang4847ae62017-06-27 07:06:52 -0700972 void WaitForEncodedFrame(int64_t expected_ntp_time) {
973 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200974 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700975 }
976
977 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
978 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200979 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700980 return ok;
981 }
982
983 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
984 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200985 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700986 }
987
988 void ExpectDroppedFrame() {
989 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200990 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700991 }
992
993 bool WaitForFrame(int64_t timeout_ms) {
994 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200995 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700996 return ok;
997 }
998
perkj26091b12016-09-01 01:17:40 -0700999 class TestEncoder : public test::FakeEncoder {
1000 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001001 explicit TestEncoder(TimeController* time_controller)
1002 : FakeEncoder(time_controller->GetClock()),
1003 time_controller_(time_controller) {
1004 RTC_DCHECK(time_controller_);
1005 }
perkj26091b12016-09-01 01:17:40 -07001006
Erik Språngaed30702018-11-05 12:57:17 +01001007 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001008 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001009 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001010 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001011 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001012 info.scaling_settings = VideoEncoder::ScalingSettings(
1013 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001014 }
1015 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001016 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1017 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001018 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001019 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001020 for (int tid = 0; tid < num_layers; ++tid)
1021 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001022 }
1023 }
Erik Språngaed30702018-11-05 12:57:17 +01001024 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001025
1026 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001027 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001028 info.apply_alignment_to_all_simulcast_layers =
1029 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001030 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001031 if (is_qp_trusted_.has_value()) {
1032 info.is_qp_trusted = is_qp_trusted_;
1033 }
Erik Språngaed30702018-11-05 12:57:17 +01001034 return info;
kthelgason876222f2016-11-29 01:44:11 -08001035 }
1036
Erik Språngb7cb7b52019-02-26 15:52:33 +01001037 int32_t RegisterEncodeCompleteCallback(
1038 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001039 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001040 encoded_image_callback_ = callback;
1041 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1042 }
1043
perkjfa10b552016-10-02 23:45:26 -07001044 void ContinueEncode() { continue_encode_event_.Set(); }
1045
1046 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1047 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001048 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001049 EXPECT_EQ(timestamp_, timestamp);
1050 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1051 }
1052
kthelgason2fc52542017-03-03 00:24:41 -08001053 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001055 quality_scaling_ = b;
1056 }
kthelgasonad9010c2017-02-14 00:46:51 -08001057
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001058 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001059 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001060 requested_resolution_alignment_ = requested_resolution_alignment;
1061 }
1062
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001063 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1064 MutexLock lock(&local_mutex_);
1065 apply_alignment_to_all_simulcast_layers_ = b;
1066 }
1067
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001068 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001069 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001070 is_hardware_accelerated_ = is_hardware_accelerated;
1071 }
1072
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001073 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1074 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001075 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001076 temporal_layers_supported_[spatial_idx] = supported;
1077 }
1078
Sergey Silkin6456e352019-07-08 17:56:40 +02001079 void SetResolutionBitrateLimits(
1080 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001081 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001082 resolution_bitrate_limits_ = thresholds;
1083 }
1084
sprangfe627f32017-03-29 08:24:59 -07001085 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001086 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001087 force_init_encode_failed_ = force_failure;
1088 }
1089
Niels Möller6bb5ab92019-01-11 11:11:10 +01001090 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001091 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001092 rate_factor_ = rate_factor;
1093 }
1094
Erik Språngd7329ca2019-02-21 21:19:53 +01001095 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001096 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001097 return last_framerate_;
1098 }
1099
Erik Språngd7329ca2019-02-21 21:19:53 +01001100 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001101 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001102 return last_update_rect_;
1103 }
1104
Niels Möller87e2d782019-03-07 10:18:23 +01001105 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001106 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001107 return last_frame_types_;
1108 }
1109
1110 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001111 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001112 keyframe ? VideoFrameType::kVideoFrameKey
1113 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001114 {
Markus Handella3765182020-07-08 13:13:32 +02001115 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001116 last_frame_types_ = frame_type;
1117 }
Niels Möllerb859b322019-03-07 12:40:01 +01001118 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001119 }
1120
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001121 void InjectEncodedImage(const EncodedImage& image,
1122 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001123 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001124 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001125 }
1126
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001127 void SetEncodedImageData(
1128 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001129 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001130 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001131 }
1132
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001134 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001135 expect_null_frame_ = true;
1136 }
1137
Erik Språng5056af02019-09-02 15:53:11 +02001138 absl::optional<VideoEncoder::RateControlParameters>
1139 GetAndResetLastRateControlSettings() {
1140 auto settings = last_rate_control_settings_;
1141 last_rate_control_settings_.reset();
1142 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001143 }
1144
Henrik Boström56db9ff2021-03-24 09:06:45 +01001145 int GetLastInputWidth() const {
1146 MutexLock lock(&local_mutex_);
1147 return last_input_width_;
1148 }
1149
1150 int GetLastInputHeight() const {
1151 MutexLock lock(&local_mutex_);
1152 return last_input_height_;
1153 }
1154
Evan Shrubsole895556e2020-10-05 09:15:13 +02001155 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1156 MutexLock lock(&local_mutex_);
1157 return last_input_pixel_format_;
1158 }
1159
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001160 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001161 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001162 return num_set_rates_;
1163 }
1164
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001165 void SetPreferredPixelFormats(
1166 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1167 pixel_formats) {
1168 MutexLock lock(&local_mutex_);
1169 preferred_pixel_formats_ = std::move(pixel_formats);
1170 }
1171
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001172 void SetIsQpTrusted(absl::optional<bool> trusted) {
1173 MutexLock lock(&local_mutex_);
1174 is_qp_trusted_ = trusted;
1175 }
1176
perkjfa10b552016-10-02 23:45:26 -07001177 private:
perkj26091b12016-09-01 01:17:40 -07001178 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001179 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001180 {
Markus Handella3765182020-07-08 13:13:32 +02001181 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001182 if (expect_null_frame_) {
1183 EXPECT_EQ(input_image.timestamp(), 0u);
1184 EXPECT_EQ(input_image.width(), 1);
1185 last_frame_types_ = *frame_types;
1186 expect_null_frame_ = false;
1187 } else {
1188 EXPECT_GT(input_image.timestamp(), timestamp_);
1189 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1190 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1191 }
perkj26091b12016-09-01 01:17:40 -07001192
1193 timestamp_ = input_image.timestamp();
1194 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001195 last_input_width_ = input_image.width();
1196 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001197 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001198 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001199 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001200 }
Niels Möllerb859b322019-03-07 12:40:01 +01001201 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001202 return result;
1203 }
1204
Niels Möller08ae7ce2020-09-23 15:58:12 +02001205 CodecSpecificInfo EncodeHook(
1206 EncodedImage& encoded_image,
1207 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001208 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001209 {
1210 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001211 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001212 }
1213 MutexLock lock(&local_mutex_);
1214 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001215 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001216 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001217 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001218 }
1219
sprangfe627f32017-03-29 08:24:59 -07001220 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001221 const Settings& settings) override {
1222 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001223
Markus Handella3765182020-07-08 13:13:32 +02001224 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001225 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001226
Erik Språng82fad3d2018-03-21 09:57:23 +01001227 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001228 // Simulate setting up temporal layers, in order to validate the life
1229 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001230 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001231 frame_buffer_controller_ =
1232 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001233 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001234 if (force_init_encode_failed_) {
1235 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001236 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001237 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001238
Erik Språngb7cb7b52019-02-26 15:52:33 +01001239 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001240 return res;
1241 }
1242
Erik Språngb7cb7b52019-02-26 15:52:33 +01001243 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001244 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001245 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1246 initialized_ = EncoderState::kUninitialized;
1247 return FakeEncoder::Release();
1248 }
1249
Erik Språng16cb8f52019-04-12 13:59:09 +02001250 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001251 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001252 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001253 VideoBitrateAllocation adjusted_rate_allocation;
1254 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1255 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001256 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001257 adjusted_rate_allocation.SetBitrate(
1258 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001259 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001260 rate_factor_));
1261 }
1262 }
1263 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001264 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001265 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001266 RateControlParameters adjusted_paramters = parameters;
1267 adjusted_paramters.bitrate = adjusted_rate_allocation;
1268 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001269 }
1270
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001271 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001272 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001273 enum class EncoderState {
1274 kUninitialized,
1275 kInitializationFailed,
1276 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001277 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001278 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001279 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1280 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1281 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1282 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1283 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1284 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001285 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1286 false;
Markus Handella3765182020-07-08 13:13:32 +02001287 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001288 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1289 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001290 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001291 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001292 absl::optional<bool>
1293 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001294 local_mutex_);
1295 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1296 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1297 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001298 absl::optional<VideoEncoder::RateControlParameters>
1299 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001300 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1301 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001302 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001303 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001304 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1305 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001306 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001307 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001308 RTC_GUARDED_BY(local_mutex_);
1309 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001310 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1311 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001312 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1313 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001314 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001315 };
1316
mflodmancc3d4422017-08-03 08:27:51 -07001317 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001318 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001319 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1320 : time_controller_(time_controller), test_encoder_(test_encoder) {
1321 RTC_DCHECK(time_controller_);
1322 }
perkj26091b12016-09-01 01:17:40 -07001323
perkj26091b12016-09-01 01:17:40 -07001324 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001325 EXPECT_TRUE(
1326 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1327 }
1328
1329 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1330 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001331 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001332 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001333 return false;
perkj26091b12016-09-01 01:17:40 -07001334 {
Markus Handella3765182020-07-08 13:13:32 +02001335 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001336 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001337 }
1338 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001339 return true;
perkj26091b12016-09-01 01:17:40 -07001340 }
1341
sprangb1ca0732017-02-01 08:38:12 -08001342 void WaitForEncodedFrame(uint32_t expected_width,
1343 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001344 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001345 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001346 }
1347
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001348 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001349 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001350 uint32_t width = 0;
1351 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001352 {
Markus Handella3765182020-07-08 13:13:32 +02001353 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001354 width = last_width_;
1355 height = last_height_;
1356 }
1357 EXPECT_EQ(expected_height, height);
1358 EXPECT_EQ(expected_width, width);
1359 }
1360
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001361 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1362 VideoRotation rotation;
1363 {
Markus Handella3765182020-07-08 13:13:32 +02001364 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001365 rotation = last_rotation_;
1366 }
1367 EXPECT_EQ(expected_rotation, rotation);
1368 }
1369
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001370 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001371
sprangc5d62e22017-04-02 23:53:04 -07001372 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001373 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001374 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001375 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001376 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001377 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001378 }
1379
perkj26091b12016-09-01 01:17:40 -07001380 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001381 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001382 expect_frames_ = false;
1383 }
1384
asaperssonfab67072017-04-04 05:51:49 -07001385 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001386 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001387 return number_of_reconfigurations_;
1388 }
1389
asaperssonfab67072017-04-04 05:51:49 -07001390 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001391 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001392 return min_transmit_bitrate_bps_;
1393 }
1394
Erik Språngd7329ca2019-02-21 21:19:53 +01001395 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001396 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001397 num_expected_layers_ = num_layers;
1398 }
1399
Erik Språngb7cb7b52019-02-26 15:52:33 +01001400 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001401 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001402 return last_capture_time_ms_;
1403 }
1404
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001405 const EncodedImage& GetLastEncodedImage() {
1406 MutexLock lock(&mutex_);
1407 return last_encoded_image_;
1408 }
1409
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001410 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001411 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001412 return std::move(last_encoded_image_data_);
1413 }
1414
Per Kjellanderdcef6412020-10-07 15:09:05 +02001415 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1416 MutexLock lock(&mutex_);
1417 return last_bitrate_allocation_;
1418 }
1419
1420 int number_of_bitrate_allocations() const {
1421 MutexLock lock(&mutex_);
1422 return number_of_bitrate_allocations_;
1423 }
1424
Per Kjellandera9434842020-10-15 17:53:22 +02001425 VideoLayersAllocation GetLastVideoLayersAllocation() {
1426 MutexLock lock(&mutex_);
1427 return last_layers_allocation_;
1428 }
1429
1430 int number_of_layers_allocations() const {
1431 MutexLock lock(&mutex_);
1432 return number_of_layers_allocations_;
1433 }
1434
perkj26091b12016-09-01 01:17:40 -07001435 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001436 Result OnEncodedImage(
1437 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001438 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001439 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001440 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001441 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001442 last_encoded_image_data_ = std::vector<uint8_t>(
1443 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001444 uint32_t timestamp = encoded_image.Timestamp();
1445 if (last_timestamp_ != timestamp) {
1446 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001447 last_width_ = encoded_image._encodedWidth;
1448 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001449 } else {
1450 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001451 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1452 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001453 }
1454 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001455 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001456 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001457 if (num_received_layers_ == num_expected_layers_) {
1458 encoded_frame_event_.Set();
1459 }
sprangb1ca0732017-02-01 08:38:12 -08001460 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001461 }
1462
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001463 void OnEncoderConfigurationChanged(
1464 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001465 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001466 VideoEncoderConfig::ContentType content_type,
1467 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001468 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001469 ++number_of_reconfigurations_;
1470 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1471 }
1472
Per Kjellanderdcef6412020-10-07 15:09:05 +02001473 void OnBitrateAllocationUpdated(
1474 const VideoBitrateAllocation& allocation) override {
1475 MutexLock lock(&mutex_);
1476 ++number_of_bitrate_allocations_;
1477 last_bitrate_allocation_ = allocation;
1478 }
1479
Per Kjellandera9434842020-10-15 17:53:22 +02001480 void OnVideoLayersAllocationUpdated(
1481 VideoLayersAllocation allocation) override {
1482 MutexLock lock(&mutex_);
1483 ++number_of_layers_allocations_;
1484 last_layers_allocation_ = allocation;
1485 rtc::StringBuilder log;
1486 for (const auto& layer : allocation.active_spatial_layers) {
1487 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1488 << "[";
1489 for (const auto target_bitrate :
1490 layer.target_bitrate_per_temporal_layer) {
1491 log << target_bitrate.kbps() << ",";
1492 }
1493 log << "]";
1494 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001495 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001496 }
1497
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001498 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001499 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001500 TestEncoder* test_encoder_;
1501 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001502 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001503 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001504 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001505 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001506 uint32_t last_height_ = 0;
1507 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001508 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001509 size_t num_expected_layers_ = 1;
1510 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001511 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001512 int number_of_reconfigurations_ = 0;
1513 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001514 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1515 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001516 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1517 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001518 };
1519
Sergey Silkin5ee69672019-07-02 14:18:34 +02001520 class VideoBitrateAllocatorProxyFactory
1521 : public VideoBitrateAllocatorFactory {
1522 public:
1523 VideoBitrateAllocatorProxyFactory()
1524 : bitrate_allocator_factory_(
1525 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1526
1527 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1528 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001529 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001530 codec_config_ = codec;
1531 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1532 }
1533
1534 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001535 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001536 return codec_config_;
1537 }
1538
1539 private:
1540 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1541
Markus Handella3765182020-07-08 13:13:32 +02001542 mutable Mutex mutex_;
1543 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001544 };
1545
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001546 Clock* clock() { return time_controller_.GetClock(); }
1547 void AdvanceTime(TimeDelta duration) {
1548 time_controller_.AdvanceTime(duration);
1549 }
1550
1551 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1552
1553 protected:
1554 virtual TaskQueueFactory* GetTaskQueueFactory() {
1555 return time_controller_.GetTaskQueueFactory();
1556 }
1557
1558 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001559 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001560 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001561 int codec_width_;
1562 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001563 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001564 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001565 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001566 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001567 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001568 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001569 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001570 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001571};
1572
mflodmancc3d4422017-08-03 08:27:51 -07001573TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001575 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001576 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001577 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001578 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001579 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001580 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001581}
1582
mflodmancc3d4422017-08-03 08:27:51 -07001583TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001584 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001585 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001586 // The encoder will cache up to one frame for a short duration. Adding two
1587 // frames means that the first frame will be dropped and the second frame will
1588 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001589 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001590 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001591 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001592 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001593 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001594
Henrik Boström381d1092020-05-12 18:49:07 +02001595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001596 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001597
Sebastian Janssona3177052018-04-10 13:05:49 +02001598 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001599 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001600 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1601
1602 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001603 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001604}
1605
mflodmancc3d4422017-08-03 08:27:51 -07001606TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001608 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001609 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001610 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001611
Henrik Boström381d1092020-05-12 18:49:07 +02001612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001613 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1614
Sebastian Janssona3177052018-04-10 13:05:49 +02001615 // The encoder will cache up to one frame for a short duration. Adding two
1616 // frames means that the first frame will be dropped and the second frame will
1617 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001618 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001619 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001620
Henrik Boström381d1092020-05-12 18:49:07 +02001621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001622 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001623 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001624 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1625 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001626 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001627}
1628
mflodmancc3d4422017-08-03 08:27:51 -07001629TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001631 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001632 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001633 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001634
1635 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001636 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001637
perkja49cbd32016-09-16 07:53:41 -07001638 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001639 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001640 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001641}
1642
mflodmancc3d4422017-08-03 08:27:51 -07001643TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001646
perkja49cbd32016-09-16 07:53:41 -07001647 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001648 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001649
mflodmancc3d4422017-08-03 08:27:51 -07001650 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001651 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001652 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001653 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1654 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001655}
1656
Markus Handell9a478b52021-11-18 16:07:01 +01001657TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1658 test::FrameForwarder source;
1659 video_stream_encoder_->SetSource(&source,
1660 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001662 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001663
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001664 int dropped_count = 0;
1665 stats_proxy_->SetDroppedFrameCallback(
1666 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1667 ++dropped_count;
1668 });
1669
Markus Handell9a478b52021-11-18 16:07:01 +01001670 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1671 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1672 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001673 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001674 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001675}
1676
Henrik Boström56db9ff2021-03-24 09:06:45 +01001677TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001679 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001680
1681 rtc::Event frame_destroyed_event;
1682 video_source_.IncomingCapturedFrame(
1683 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001684 WaitForEncodedFrame(1);
1685 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1686 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001687 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1688 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001689 video_stream_encoder_->Stop();
1690}
1691
Henrik Boström56db9ff2021-03-24 09:06:45 +01001692TEST_F(VideoStreamEncoderTest,
1693 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001694 // Use the cropping factory.
1695 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001696 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001697 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1698 kMaxPayloadLength);
1699 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1700
1701 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001703 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001704 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1705 WaitForEncodedFrame(1);
1706 // The encoder will have been configured once.
1707 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001708 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1709 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001710
1711 // Now send in a fake frame that needs to be cropped as the width/height
1712 // aren't divisible by 4 (see CreateEncoderStreams above).
1713 rtc::Event frame_destroyed_event;
1714 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1715 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001716 WaitForEncodedFrame(2);
1717 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1718 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001719 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1720 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001721 video_stream_encoder_->Stop();
1722}
1723
Evan Shrubsole895556e2020-10-05 09:15:13 +02001724TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001726 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001727
1728 video_source_.IncomingCapturedFrame(
1729 CreateNV12Frame(1, codec_width_, codec_height_));
1730 WaitForEncodedFrame(1);
1731 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1732 fake_encoder_.GetLastInputPixelFormat());
1733 video_stream_encoder_->Stop();
1734}
1735
Henrik Boström56db9ff2021-03-24 09:06:45 +01001736TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001738 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001739
1740 fake_encoder_.SetPreferredPixelFormats({});
1741
1742 rtc::Event frame_destroyed_event;
1743 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1744 1, &frame_destroyed_event, codec_width_, codec_height_));
1745 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001746 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001747 fake_encoder_.GetLastInputPixelFormat());
1748 video_stream_encoder_->Stop();
1749}
1750
Henrik Boström56db9ff2021-03-24 09:06:45 +01001751TEST_F(VideoStreamEncoderTest,
1752 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001755
1756 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1757
1758 rtc::Event frame_destroyed_event;
1759 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1760 1, &frame_destroyed_event, codec_width_, codec_height_));
1761 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001762 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001763 fake_encoder_.GetLastInputPixelFormat());
1764 video_stream_encoder_->Stop();
1765}
1766
Henrik Boström56db9ff2021-03-24 09:06:45 +01001767TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001769 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001770
1771 // Fake NV12 native frame does not allow mapping to I444.
1772 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1773
1774 rtc::Event frame_destroyed_event;
1775 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1776 1, &frame_destroyed_event, codec_width_, codec_height_));
1777 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001778 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001779 fake_encoder_.GetLastInputPixelFormat());
1780 video_stream_encoder_->Stop();
1781}
1782
Henrik Boström56db9ff2021-03-24 09:06:45 +01001783TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001786
1787 rtc::Event frame_destroyed_event;
1788 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1789 1, &frame_destroyed_event, codec_width_, codec_height_));
1790 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001791 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001792 fake_encoder_.GetLastInputPixelFormat());
1793 video_stream_encoder_->Stop();
1794}
1795
Ying Wang9b881ab2020-02-07 14:29:32 +01001796TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001798 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001799 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1800 WaitForEncodedFrame(1);
1801
Henrik Boström381d1092020-05-12 18:49:07 +02001802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001803 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001804 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1805 // frames. Adding two frames means that the first frame will be dropped and
1806 // the second frame will be sent to the encoder.
1807 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1808 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1809 WaitForEncodedFrame(3);
1810 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1811 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1812 WaitForEncodedFrame(5);
1813 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1814 video_stream_encoder_->Stop();
1815}
1816
mflodmancc3d4422017-08-03 08:27:51 -07001817TEST_F(VideoStreamEncoderTest,
1818 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001820 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001821 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001822
1823 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001824 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001825 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001826 // The encoder will have been configured once when the first frame is
1827 // received.
1828 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001829
1830 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001831 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001832 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001833 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001834 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001835
1836 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001837 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001838 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001839 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001840 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001841
mflodmancc3d4422017-08-03 08:27:51 -07001842 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001843}
1844
mflodmancc3d4422017-08-03 08:27:51 -07001845TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001848
1849 // Capture a frame and wait for it to synchronize with the encoder thread.
1850 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001851 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001852 // The encoder will have been configured once.
1853 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001854 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1855 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001856
1857 codec_width_ *= 2;
1858 codec_height_ *= 2;
1859 // Capture a frame with a higher resolution and wait for it to synchronize
1860 // with the encoder thread.
1861 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001862 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001863 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1864 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001865 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001866
mflodmancc3d4422017-08-03 08:27:51 -07001867 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001868}
1869
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001870TEST_F(VideoStreamEncoderTest,
1871 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001873 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001874
1875 // Capture a frame and wait for it to synchronize with the encoder thread.
1876 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1877 WaitForEncodedFrame(1);
1878
1879 VideoEncoderConfig video_encoder_config;
1880 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1881 // Changing the max payload data length recreates encoder.
1882 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1883 kMaxPayloadLength / 2);
1884
1885 // Capture a frame and wait for it to synchronize with the encoder thread.
1886 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1887 WaitForEncodedFrame(2);
1888 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1889
1890 video_stream_encoder_->Stop();
1891}
1892
Sergey Silkin5ee69672019-07-02 14:18:34 +02001893TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001894 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001895 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001896
1897 VideoEncoderConfig video_encoder_config;
1898 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001899 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1900 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001901 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1902 kMaxPayloadLength);
1903
1904 // Capture a frame and wait for it to synchronize with the encoder thread.
1905 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1906 WaitForEncodedFrame(1);
1907 // The encoder will have been configured once when the first frame is
1908 // received.
1909 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001910 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001911 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001912 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001913 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1914
Sergey Silkin6456e352019-07-08 17:56:40 +02001915 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1916 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001917 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1918 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001919 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1920 kMaxPayloadLength);
1921
1922 // Capture a frame and wait for it to synchronize with the encoder thread.
1923 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1924 WaitForEncodedFrame(2);
1925 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1926 // Bitrate limits have changed - rate allocator should be reconfigured,
1927 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001928 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001929 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001930 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001931 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001932 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001933
1934 video_stream_encoder_->Stop();
1935}
1936
Sergey Silkin6456e352019-07-08 17:56:40 +02001937TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001938 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001940 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001941
Sergey Silkincd02eba2020-01-20 14:48:40 +01001942 const uint32_t kMinEncBitrateKbps = 100;
1943 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001944 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001945 /*frame_size_pixels=*/codec_width_ * codec_height_,
1946 /*min_start_bitrate_bps=*/0,
1947 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1948 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1950
Sergey Silkincd02eba2020-01-20 14:48:40 +01001951 VideoEncoderConfig video_encoder_config;
1952 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1953 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1954 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1955 (kMinEncBitrateKbps + 1) * 1000;
1956 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1957 kMaxPayloadLength);
1958
1959 // When both encoder and app provide bitrate limits, the intersection of
1960 // provided sets should be used.
1961 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1962 WaitForEncodedFrame(1);
1963 EXPECT_EQ(kMaxEncBitrateKbps,
1964 bitrate_allocator_factory_.codec_config().maxBitrate);
1965 EXPECT_EQ(kMinEncBitrateKbps + 1,
1966 bitrate_allocator_factory_.codec_config().minBitrate);
1967
1968 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1969 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1970 (kMinEncBitrateKbps - 1) * 1000;
1971 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1972 kMaxPayloadLength);
1973 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001974 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001975 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001976 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001977 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001978 bitrate_allocator_factory_.codec_config().minBitrate);
1979
Sergey Silkincd02eba2020-01-20 14:48:40 +01001980 video_stream_encoder_->Stop();
1981}
1982
1983TEST_F(VideoStreamEncoderTest,
1984 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001986 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001987
1988 const uint32_t kMinAppBitrateKbps = 100;
1989 const uint32_t kMaxAppBitrateKbps = 200;
1990 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1991 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1992 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1993 /*frame_size_pixels=*/codec_width_ * codec_height_,
1994 /*min_start_bitrate_bps=*/0,
1995 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1996 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1997 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1998
1999 VideoEncoderConfig video_encoder_config;
2000 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2001 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2002 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2003 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2005 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002006
Sergey Silkincd02eba2020-01-20 14:48:40 +01002007 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2008 WaitForEncodedFrame(1);
2009 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002010 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002011 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002012 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002013
2014 video_stream_encoder_->Stop();
2015}
2016
2017TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002018 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002020 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002021
2022 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002024 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002026 fake_encoder_.SetResolutionBitrateLimits(
2027 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2028
2029 VideoEncoderConfig video_encoder_config;
2030 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2031 video_encoder_config.max_bitrate_bps = 0;
2032 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2033 kMaxPayloadLength);
2034
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002035 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002036 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2037 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002038 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2039 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002040 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2041 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2042
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002043 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002044 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2045 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002046 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2047 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002048 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2049 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2050
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002051 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002052 // encoder for 360p should be used.
2053 video_source_.IncomingCapturedFrame(
2054 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2055 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002056 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2057 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002058 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2059 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2060
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002061 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002062 // ignored.
2063 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2064 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002065 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2066 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002067 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2068 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002069 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2070 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002071 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2072 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2073
2074 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2075 // for 270p should be used.
2076 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2077 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002078 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2079 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002080 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2081 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2082
2083 video_stream_encoder_->Stop();
2084}
2085
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002086TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002088 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002089
2090 VideoEncoderConfig video_encoder_config;
2091 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2092 video_encoder_config.max_bitrate_bps = 0;
2093 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2094 kMaxPayloadLength);
2095
2096 // Encode 720p frame to get the default encoder target bitrate.
2097 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2098 WaitForEncodedFrame(1);
2099 const uint32_t kDefaultTargetBitrateFor720pKbps =
2100 bitrate_allocator_factory_.codec_config()
2101 .simulcastStream[0]
2102 .targetBitrate;
2103
2104 // Set the max recommended encoder bitrate to something lower than the default
2105 // target bitrate.
2106 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2107 1280 * 720, 10 * 1000, 10 * 1000,
2108 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2109 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2110
2111 // Change resolution to trigger encoder reinitialization.
2112 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2113 WaitForEncodedFrame(2);
2114 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2115 WaitForEncodedFrame(3);
2116
2117 // Ensure the target bitrate is capped by the max bitrate.
2118 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2119 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2120 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2121 .simulcastStream[0]
2122 .targetBitrate *
2123 1000,
2124 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2125
2126 video_stream_encoder_->Stop();
2127}
2128
Åsa Perssona7e34d32021-01-20 15:36:13 +01002129TEST_F(VideoStreamEncoderTest,
2130 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2131 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2132 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2133 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2134 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2135 fake_encoder_.SetResolutionBitrateLimits(
2136 {kEncoderLimits270p, kEncoderLimits360p});
2137
2138 // Two streams, highest stream active.
2139 VideoEncoderConfig config;
2140 const int kNumStreams = 2;
2141 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2142 config.max_bitrate_bps = 0;
2143 config.simulcast_layers[0].active = false;
2144 config.simulcast_layers[1].active = true;
2145 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002146 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002147 "VP8", /*max qp*/ 56, /*screencast*/ false,
2148 /*screenshare enabled*/ false);
2149 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2150
2151 // The encoder bitrate limits for 270p should be used.
2152 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2153 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002154 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002155 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002156 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002157 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002158 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002159
2160 // The encoder bitrate limits for 360p should be used.
2161 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2162 EXPECT_FALSE(WaitForFrame(1000));
2163 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002164 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002165 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002166 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002167
2168 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2169 video_source_.IncomingCapturedFrame(
2170 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2171 EXPECT_FALSE(WaitForFrame(1000));
2172 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002173 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002174 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002175 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002176
2177 // Resolution higher than 360p. Encoder limits should be ignored.
2178 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2179 EXPECT_FALSE(WaitForFrame(1000));
2180 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002181 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002182 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002183 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002184 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002185 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002186 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002187 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002188
2189 // Resolution lower than 270p. The encoder limits for 270p should be used.
2190 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2191 EXPECT_FALSE(WaitForFrame(1000));
2192 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002193 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002194 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002195 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002196
2197 video_stream_encoder_->Stop();
2198}
2199
2200TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002201 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2202 // Two streams, highest stream active.
2203 VideoEncoderConfig config;
2204 const int kNumStreams = 2;
2205 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2206 config.max_bitrate_bps = 0;
2207 config.simulcast_layers[0].active = false;
2208 config.simulcast_layers[1].active = true;
2209 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002210 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002211 "VP8", /*max qp*/ 56, /*screencast*/ false,
2212 /*screenshare enabled*/ false);
2213 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2214
2215 // Default bitrate limits for 270p should be used.
2216 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2217 kDefaultLimits270p =
2218 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002219 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002220 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2221 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002223 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002225 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002226 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002227
2228 // Default bitrate limits for 360p should be used.
2229 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2230 kDefaultLimits360p =
2231 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002232 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002233 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2234 EXPECT_FALSE(WaitForFrame(1000));
2235 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002237 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002238 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002239
2240 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2241 video_source_.IncomingCapturedFrame(
2242 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2243 EXPECT_FALSE(WaitForFrame(1000));
2244 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002245 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002246 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002247 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002248
2249 // Default bitrate limits for 540p should be used.
2250 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2251 kDefaultLimits540p =
2252 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002253 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002254 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2255 EXPECT_FALSE(WaitForFrame(1000));
2256 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002257 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002258 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002259 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002260
2261 video_stream_encoder_->Stop();
2262}
2263
2264TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002265 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2266 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2267 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2268 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2269 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2270 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2271 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2272 fake_encoder_.SetResolutionBitrateLimits(
2273 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2274
2275 // Three streams, middle stream active.
2276 VideoEncoderConfig config;
2277 const int kNumStreams = 3;
2278 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2279 config.simulcast_layers[0].active = false;
2280 config.simulcast_layers[1].active = true;
2281 config.simulcast_layers[2].active = false;
2282 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002283 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002284 "VP8", /*max qp*/ 56, /*screencast*/ false,
2285 /*screenshare enabled*/ false);
2286 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2287
2288 // The encoder bitrate limits for 360p should be used.
2289 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2290 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002291 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002292 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002293 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002294 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002295 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002296
2297 // The encoder bitrate limits for 270p should be used.
2298 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2299 EXPECT_FALSE(WaitForFrame(1000));
2300 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002301 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002302 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002303 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002304
2305 video_stream_encoder_->Stop();
2306}
2307
2308TEST_F(VideoStreamEncoderTest,
2309 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2310 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2311 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2312 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2313 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2314 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2315 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2316 fake_encoder_.SetResolutionBitrateLimits(
2317 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2318
2319 // Three streams, lowest stream active.
2320 VideoEncoderConfig config;
2321 const int kNumStreams = 3;
2322 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2323 config.simulcast_layers[0].active = true;
2324 config.simulcast_layers[1].active = false;
2325 config.simulcast_layers[2].active = false;
2326 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002327 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002328 "VP8", /*max qp*/ 56, /*screencast*/ false,
2329 /*screenshare enabled*/ false);
2330 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2331
2332 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2333 // on lowest stream, limits for 270p should not be used
2334 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2335 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002336 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002337 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002338 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002339 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002340 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341
2342 video_stream_encoder_->Stop();
2343}
2344
2345TEST_F(VideoStreamEncoderTest,
2346 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2347 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2348 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2349 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2350 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2351 fake_encoder_.SetResolutionBitrateLimits(
2352 {kEncoderLimits270p, kEncoderLimits360p});
2353 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2354
2355 // Two streams, highest stream active.
2356 VideoEncoderConfig config;
2357 const int kNumStreams = 2;
2358 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2359 config.simulcast_layers[0].active = false;
2360 config.simulcast_layers[1].active = true;
2361 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2362 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002363 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002364 "VP8", /*max qp*/ 56, /*screencast*/ false,
2365 /*screenshare enabled*/ false);
2366 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2367
2368 // The encoder bitrate limits for 270p should be used.
2369 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2370 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002371 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002372 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002373 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002374 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002375 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002376
2377 // The max configured bitrate is less than the encoder limit for 360p.
2378 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2379 EXPECT_FALSE(WaitForFrame(1000));
2380 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002381 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002382 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002383 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002384
2385 video_stream_encoder_->Stop();
2386}
2387
mflodmancc3d4422017-08-03 08:27:51 -07002388TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002389 EXPECT_TRUE(video_source_.has_sinks());
2390 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002391 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002392 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002393 EXPECT_FALSE(video_source_.has_sinks());
2394 EXPECT_TRUE(new_video_source.has_sinks());
2395
mflodmancc3d4422017-08-03 08:27:51 -07002396 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002397}
2398
mflodmancc3d4422017-08-03 08:27:51 -07002399TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002400 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002401 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002402 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002403 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002404}
2405
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002406class ResolutionAlignmentTest
2407 : public VideoStreamEncoderTest,
2408 public ::testing::WithParamInterface<
2409 ::testing::tuple<int, std::vector<double>>> {
2410 public:
2411 ResolutionAlignmentTest()
2412 : requested_alignment_(::testing::get<0>(GetParam())),
2413 scale_factors_(::testing::get<1>(GetParam())) {}
2414
2415 protected:
2416 const int requested_alignment_;
2417 const std::vector<double> scale_factors_;
2418};
2419
2420INSTANTIATE_TEST_SUITE_P(
2421 AlignmentAndScaleFactors,
2422 ResolutionAlignmentTest,
2423 ::testing::Combine(
2424 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2425 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2426 std::vector<double>{-1.0, -1.0},
2427 std::vector<double>{-1.0, -1.0, -1.0},
2428 std::vector<double>{4.0, 2.0, 1.0},
2429 std::vector<double>{9999.0, -1.0, 1.0},
2430 std::vector<double>{3.99, 2.01, 1.0},
2431 std::vector<double>{4.9, 1.7, 1.25},
2432 std::vector<double>{10.0, 4.0, 3.0},
2433 std::vector<double>{1.75, 3.5},
2434 std::vector<double>{1.5, 2.5},
2435 std::vector<double>{1.3, 1.0})));
2436
2437TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2438 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002439 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002440 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2441 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2442
2443 // Fill config with the scaling factor by which to reduce encoding size.
2444 const int num_streams = scale_factors_.size();
2445 VideoEncoderConfig config;
2446 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2447 for (int i = 0; i < num_streams; ++i) {
2448 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2449 }
2450 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002451 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002452 "VP8", /*max qp*/ 56, /*screencast*/ false,
2453 /*screenshare enabled*/ false);
2454 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2455
Henrik Boström381d1092020-05-12 18:49:07 +02002456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002457 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2458 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002459 // Wait for all layers before triggering event.
2460 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002461
2462 // On the 1st frame, we should have initialized the encoder and
2463 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002464 int64_t timestamp_ms = kFrameIntervalMs;
2465 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2466 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002467 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002468
2469 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2470 // (It's up the to the encoder to potentially drop the previous frame,
2471 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002472 timestamp_ms += kFrameIntervalMs;
2473 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2474 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002475 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002476
Asa Persson606d3cb2021-10-04 10:07:11 +02002477 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002478 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2479 // Frame size should be a multiple of the requested alignment.
2480 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2481 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2482 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2483 // Aspect ratio should match.
2484 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2485 codec.height * codec.simulcastStream[i].width);
2486 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002487
2488 video_stream_encoder_->Stop();
2489}
2490
Jonathan Yubc771b72017-12-08 17:04:29 -08002491TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2492 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002493 const int kWidth = 1280;
2494 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002495
2496 // We rely on the automatic resolution adaptation, but we handle framerate
2497 // adaptation manually by mocking the stats proxy.
2498 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002499
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002500 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002502 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002503 video_stream_encoder_->SetSource(&video_source_,
2504 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002505 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002506 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002507 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002508 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2509
Jonathan Yubc771b72017-12-08 17:04:29 -08002510 // Adapt down as far as possible.
2511 rtc::VideoSinkWants last_wants;
2512 int64_t t = 1;
2513 int loop_count = 0;
2514 do {
2515 ++loop_count;
2516 last_wants = video_source_.sink_wants();
2517
2518 // Simulate the framerate we've been asked to adapt to.
2519 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2520 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2521 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2522 mock_stats.input_frame_rate = fps;
2523 stats_proxy_->SetMockStats(mock_stats);
2524
2525 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2526 sink_.WaitForEncodedFrame(t);
2527 t += frame_interval_ms;
2528
mflodmancc3d4422017-08-03 08:27:51 -07002529 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002530 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002531 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002532 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2533 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002534 } while (video_source_.sink_wants().max_pixel_count <
2535 last_wants.max_pixel_count ||
2536 video_source_.sink_wants().max_framerate_fps <
2537 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002538
Jonathan Yubc771b72017-12-08 17:04:29 -08002539 // Verify that we've adapted all the way down.
2540 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002541 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002542 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2543 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002544 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002545 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2546 *video_source_.last_sent_height());
2547 EXPECT_EQ(kMinBalancedFramerateFps,
2548 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002549
Jonathan Yubc771b72017-12-08 17:04:29 -08002550 // Adapt back up the same number of times we adapted down.
2551 for (int i = 0; i < loop_count - 1; ++i) {
2552 last_wants = video_source_.sink_wants();
2553
2554 // Simulate the framerate we've been asked to adapt to.
2555 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2556 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2557 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2558 mock_stats.input_frame_rate = fps;
2559 stats_proxy_->SetMockStats(mock_stats);
2560
2561 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2562 sink_.WaitForEncodedFrame(t);
2563 t += frame_interval_ms;
2564
Henrik Boström91aa7322020-04-28 12:24:33 +02002565 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002566 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002567 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002568 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2569 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002570 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2571 last_wants.max_pixel_count ||
2572 video_source_.sink_wants().max_framerate_fps >
2573 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002574 }
2575
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002576 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002577 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002578 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002579 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2580 EXPECT_EQ((loop_count - 1) * 2,
2581 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002582
mflodmancc3d4422017-08-03 08:27:51 -07002583 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002584}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002585
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002586TEST_F(VideoStreamEncoderTest,
2587 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002588 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2589 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002590 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002591
2592 const int kFrameWidth = 1280;
2593 const int kFrameHeight = 720;
2594
2595 int64_t ntp_time = kFrameIntervalMs;
2596
2597 // Force an input frame rate to be available, or the adaptation call won't
2598 // know what framerate to adapt form.
2599 const int kInputFps = 30;
2600 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2601 stats.input_frame_rate = kInputFps;
2602 stats_proxy_->SetMockStats(stats);
2603
2604 video_source_.set_adaptation_enabled(true);
2605 video_stream_encoder_->SetSource(
2606 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002607 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002608 video_source_.IncomingCapturedFrame(
2609 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2610 sink_.WaitForEncodedFrame(ntp_time);
2611 ntp_time += kFrameIntervalMs;
2612
2613 // Trigger CPU overuse.
2614 video_stream_encoder_->TriggerCpuOveruse();
2615 video_source_.IncomingCapturedFrame(
2616 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2617 sink_.WaitForEncodedFrame(ntp_time);
2618 ntp_time += kFrameIntervalMs;
2619
2620 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2621 EXPECT_EQ(std::numeric_limits<int>::max(),
2622 video_source_.sink_wants().max_pixel_count);
2623 // Some framerate constraint should be set.
2624 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2625 EXPECT_LT(restricted_fps, kInputFps);
2626 video_source_.IncomingCapturedFrame(
2627 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2628 sink_.WaitForEncodedFrame(ntp_time);
2629 ntp_time += 100;
2630
Henrik Boström2671dac2020-05-19 16:29:09 +02002631 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002632 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2633 // Give the encoder queue time to process the change in degradation preference
2634 // by waiting for an encoded frame.
2635 video_source_.IncomingCapturedFrame(
2636 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2637 sink_.WaitForEncodedFrame(ntp_time);
2638 ntp_time += kFrameIntervalMs;
2639
2640 video_stream_encoder_->TriggerQualityLow();
2641 video_source_.IncomingCapturedFrame(
2642 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2643 sink_.WaitForEncodedFrame(ntp_time);
2644 ntp_time += kFrameIntervalMs;
2645
2646 // Some resolution constraint should be set.
2647 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2648 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2649 kFrameWidth * kFrameHeight);
2650 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2651
2652 int pixel_count = video_source_.sink_wants().max_pixel_count;
2653 // Triggering a CPU underuse should not change the sink wants since it has
2654 // not been overused for resolution since we changed degradation preference.
2655 video_stream_encoder_->TriggerCpuUnderuse();
2656 video_source_.IncomingCapturedFrame(
2657 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2658 sink_.WaitForEncodedFrame(ntp_time);
2659 ntp_time += kFrameIntervalMs;
2660 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2661 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2662
Evan Shrubsole64469032020-06-11 10:45:29 +02002663 // Change the degradation preference back. CPU underuse should not adapt since
2664 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002665 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002666 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2667 video_source_.IncomingCapturedFrame(
2668 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2669 sink_.WaitForEncodedFrame(ntp_time);
2670 ntp_time += 100;
2671 // Resolution adaptations is gone after changing degradation preference.
2672 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2673 EXPECT_EQ(std::numeric_limits<int>::max(),
2674 video_source_.sink_wants().max_pixel_count);
2675 // The fps adaptation from above is now back.
2676 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2677
2678 // Trigger CPU underuse.
2679 video_stream_encoder_->TriggerCpuUnderuse();
2680 video_source_.IncomingCapturedFrame(
2681 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2682 sink_.WaitForEncodedFrame(ntp_time);
2683 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002684 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2685
2686 // Trigger QP underuse, fps should return to normal.
2687 video_stream_encoder_->TriggerQualityHigh();
2688 video_source_.IncomingCapturedFrame(
2689 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2690 sink_.WaitForEncodedFrame(ntp_time);
2691 ntp_time += kFrameIntervalMs;
2692 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002693
2694 video_stream_encoder_->Stop();
2695}
2696
mflodmancc3d4422017-08-03 08:27:51 -07002697TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002700 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002701
sprangc5d62e22017-04-02 23:53:04 -07002702 const int kFrameWidth = 1280;
2703 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002704
Åsa Persson8c1bf952018-09-13 10:42:19 +02002705 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002706
kthelgason5e13d412016-12-01 03:59:51 -08002707 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002708 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002709 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002710 frame_timestamp += kFrameIntervalMs;
2711
perkj803d97f2016-11-01 11:45:46 -07002712 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002713 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002714 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002715 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002716 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002717 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002718
asapersson0944a802017-04-07 00:57:58 -07002719 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002720 // wanted resolution.
2721 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2722 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2723 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002724 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002725
2726 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002727 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002728 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002729 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002730 // Give the encoder queue time to process the change in degradation preference
2731 // by waiting for an encoded frame.
2732 new_video_source.IncomingCapturedFrame(
2733 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2734 sink_.WaitForEncodedFrame(frame_timestamp);
2735 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002736 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002737 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002738
sprangc5d62e22017-04-02 23:53:04 -07002739 // Force an input frame rate to be available, or the adaptation call won't
2740 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002741 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002742 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002743 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002744 stats_proxy_->SetMockStats(stats);
2745
mflodmancc3d4422017-08-03 08:27:51 -07002746 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002747 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002748 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002749 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002750 frame_timestamp += kFrameIntervalMs;
2751
2752 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002753 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002754 EXPECT_EQ(std::numeric_limits<int>::max(),
2755 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002756 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002757
asapersson02465b82017-04-10 01:12:52 -07002758 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002759 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2760 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002761 // Give the encoder queue time to process the change in degradation preference
2762 // by waiting for an encoded frame.
2763 new_video_source.IncomingCapturedFrame(
2764 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2765 sink_.WaitForEncodedFrame(frame_timestamp);
2766 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002767 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002768
mflodmancc3d4422017-08-03 08:27:51 -07002769 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002770 new_video_source.IncomingCapturedFrame(
2771 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002772 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002773 frame_timestamp += kFrameIntervalMs;
2774
2775 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002776 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002777
2778 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002779 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002780 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002781 // Give the encoder queue time to process the change in degradation preference
2782 // by waiting for an encoded frame.
2783 new_video_source.IncomingCapturedFrame(
2784 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2785 sink_.WaitForEncodedFrame(frame_timestamp);
2786 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002787 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2788 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002789 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002790 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002791
2792 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002793 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002794 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002795 // Give the encoder queue time to process the change in degradation preference
2796 // by waiting for an encoded frame.
2797 new_video_source.IncomingCapturedFrame(
2798 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2799 sink_.WaitForEncodedFrame(frame_timestamp);
2800 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002801 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2802 EXPECT_EQ(std::numeric_limits<int>::max(),
2803 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002804 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002805
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002807}
2808
mflodmancc3d4422017-08-03 08:27:51 -07002809TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002811 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002812
asaperssonfab67072017-04-04 05:51:49 -07002813 const int kWidth = 1280;
2814 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002815 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002816 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002817 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2818 EXPECT_FALSE(stats.bw_limited_resolution);
2819 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2820
2821 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002822 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002823 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002824 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002825
2826 stats = stats_proxy_->GetStats();
2827 EXPECT_TRUE(stats.bw_limited_resolution);
2828 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2829
2830 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002831 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002832 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002833 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002834
2835 stats = stats_proxy_->GetStats();
2836 EXPECT_FALSE(stats.bw_limited_resolution);
2837 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2838 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2839
mflodmancc3d4422017-08-03 08:27:51 -07002840 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002841}
2842
mflodmancc3d4422017-08-03 08:27:51 -07002843TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002844 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002845 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002846
2847 const int kWidth = 1280;
2848 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002849 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002850 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002851 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2852 EXPECT_FALSE(stats.cpu_limited_resolution);
2853 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2854
2855 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002856 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002857 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002858 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002859
2860 stats = stats_proxy_->GetStats();
2861 EXPECT_TRUE(stats.cpu_limited_resolution);
2862 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2863
2864 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002865 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002866 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002867 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002868
2869 stats = stats_proxy_->GetStats();
2870 EXPECT_FALSE(stats.cpu_limited_resolution);
2871 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002872 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002873
mflodmancc3d4422017-08-03 08:27:51 -07002874 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002875}
2876
mflodmancc3d4422017-08-03 08:27:51 -07002877TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002878 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002879 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002880
asaperssonfab67072017-04-04 05:51:49 -07002881 const int kWidth = 1280;
2882 const int kHeight = 720;
2883 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002884 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002885 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002886 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002887 EXPECT_FALSE(stats.cpu_limited_resolution);
2888 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2889
asaperssonfab67072017-04-04 05:51:49 -07002890 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002891 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002892 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002893 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002894 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002895 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002896 EXPECT_TRUE(stats.cpu_limited_resolution);
2897 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2898
2899 // Set new source with adaptation still enabled.
2900 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002901 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002902 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002903
asaperssonfab67072017-04-04 05:51:49 -07002904 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002905 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002906 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002907 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002908 EXPECT_TRUE(stats.cpu_limited_resolution);
2909 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2910
2911 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002912 video_stream_encoder_->SetSource(&new_video_source,
2913 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002914
asaperssonfab67072017-04-04 05:51:49 -07002915 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002916 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002917 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002918 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002919 EXPECT_FALSE(stats.cpu_limited_resolution);
2920 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2921
2922 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002924 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002925
asaperssonfab67072017-04-04 05:51:49 -07002926 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002927 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002928 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002929 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002930 EXPECT_TRUE(stats.cpu_limited_resolution);
2931 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2932
asaperssonfab67072017-04-04 05:51:49 -07002933 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002934 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002935 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002936 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002937 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002938 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002939 EXPECT_FALSE(stats.cpu_limited_resolution);
2940 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002941 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002942
mflodmancc3d4422017-08-03 08:27:51 -07002943 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002944}
2945
mflodmancc3d4422017-08-03 08:27:51 -07002946TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002948 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002949
asaperssonfab67072017-04-04 05:51:49 -07002950 const int kWidth = 1280;
2951 const int kHeight = 720;
2952 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002953 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002954 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002955 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002956 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002957 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002958
2959 // Set new source with adaptation still enabled.
2960 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002961 video_stream_encoder_->SetSource(&new_video_source,
2962 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002963
asaperssonfab67072017-04-04 05:51:49 -07002964 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002965 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002966 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002967 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002968 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002969 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002970
asaperssonfab67072017-04-04 05:51:49 -07002971 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002972 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002973 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002974 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002975 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002976 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002977 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002978 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002979
asaperssonfab67072017-04-04 05:51:49 -07002980 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002981 video_stream_encoder_->SetSource(&new_video_source,
2982 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002983
asaperssonfab67072017-04-04 05:51:49 -07002984 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002985 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002986 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002987 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002988 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002989 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002990
asapersson02465b82017-04-10 01:12:52 -07002991 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002992 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002993 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002994
asaperssonfab67072017-04-04 05:51:49 -07002995 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002996 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002997 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002998 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002999 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003000 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3001 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003002
mflodmancc3d4422017-08-03 08:27:51 -07003003 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003004}
3005
mflodmancc3d4422017-08-03 08:27:51 -07003006TEST_F(VideoStreamEncoderTest,
3007 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003009 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003010
3011 const int kWidth = 1280;
3012 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003013 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003014 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003015 video_source_.IncomingCapturedFrame(
3016 CreateFrame(timestamp_ms, kWidth, kHeight));
3017 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003018 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3019 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3020 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3021
3022 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003023 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003024 timestamp_ms += kFrameIntervalMs;
3025 video_source_.IncomingCapturedFrame(
3026 CreateFrame(timestamp_ms, kWidth, kHeight));
3027 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003028 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3029 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3030 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3031
3032 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003033 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003034 timestamp_ms += kFrameIntervalMs;
3035 video_source_.IncomingCapturedFrame(
3036 CreateFrame(timestamp_ms, kWidth, kHeight));
3037 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003038 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3039 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3040 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3041
Niels Möller4db138e2018-04-19 09:04:13 +02003042 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003043 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003044
3045 VideoEncoderConfig video_encoder_config;
3046 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3047 // Make format different, to force recreation of encoder.
3048 video_encoder_config.video_format.parameters["foo"] = "foo";
3049 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003050 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003051 timestamp_ms += kFrameIntervalMs;
3052 video_source_.IncomingCapturedFrame(
3053 CreateFrame(timestamp_ms, kWidth, kHeight));
3054 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003055 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3056 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3057 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3058
mflodmancc3d4422017-08-03 08:27:51 -07003059 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003060}
3061
mflodmancc3d4422017-08-03 08:27:51 -07003062TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003063 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003064 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003065 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003066
3067 const int kWidth = 1280;
3068 const int kHeight = 720;
3069 int sequence = 1;
3070
3071 // Enable BALANCED preference, no initial limitation.
3072 test::FrameForwarder source;
3073 video_stream_encoder_->SetSource(&source,
3074 webrtc::DegradationPreference::BALANCED);
3075 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3076 WaitForEncodedFrame(sequence++);
3077 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3078 EXPECT_FALSE(stats.cpu_limited_resolution);
3079 EXPECT_FALSE(stats.cpu_limited_framerate);
3080 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3081
3082 // Trigger CPU overuse, should now adapt down.
3083 video_stream_encoder_->TriggerCpuOveruse();
3084 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3085 WaitForEncodedFrame(sequence++);
3086 stats = stats_proxy_->GetStats();
3087 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3088
3089 // Set new degradation preference should clear restrictions since we changed
3090 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003091 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003092 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3093 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3094 WaitForEncodedFrame(sequence++);
3095 stats = stats_proxy_->GetStats();
3096 EXPECT_FALSE(stats.cpu_limited_resolution);
3097 EXPECT_FALSE(stats.cpu_limited_framerate);
3098 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3099
3100 // Force an input frame rate to be available, or the adaptation call won't
3101 // know what framerate to adapt from.
3102 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3103 mock_stats.input_frame_rate = 30;
3104 stats_proxy_->SetMockStats(mock_stats);
3105 video_stream_encoder_->TriggerCpuOveruse();
3106 stats_proxy_->ResetMockStats();
3107 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3108 WaitForEncodedFrame(sequence++);
3109
3110 // We have now adapted once.
3111 stats = stats_proxy_->GetStats();
3112 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3113
3114 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003115 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3116 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003117 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3118 WaitForEncodedFrame(sequence++);
3119 stats = stats_proxy_->GetStats();
3120 EXPECT_FALSE(stats.cpu_limited_resolution);
3121 EXPECT_FALSE(stats.cpu_limited_framerate);
3122 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3123
3124 video_stream_encoder_->Stop();
3125}
3126
3127TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003128 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003129 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003130 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003131
asapersson0944a802017-04-07 00:57:58 -07003132 const int kWidth = 1280;
3133 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003134 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003135
asaperssonfab67072017-04-04 05:51:49 -07003136 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003137 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003138 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003139 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003140 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003141 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3142
asapersson02465b82017-04-10 01:12:52 -07003143 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003144 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003145 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003146 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003147 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003148 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003149 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003150 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3151
3152 // Set new source with adaptation still enabled.
3153 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003154 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003155 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003156
3157 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003158 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003159 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003160 stats = stats_proxy_->GetStats();
3161 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003162 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003163 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3164
sprangc5d62e22017-04-02 23:53:04 -07003165 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003166 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003167 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003168 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003169 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003170 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003171 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003172 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003173 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003174 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003175 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3176
sprangc5d62e22017-04-02 23:53:04 -07003177 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003178 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003179 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3180 mock_stats.input_frame_rate = 30;
3181 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003182 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003183 stats_proxy_->ResetMockStats();
3184
3185 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003186 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003187 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003188
3189 // Framerate now adapted.
3190 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003191 EXPECT_FALSE(stats.cpu_limited_resolution);
3192 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003193 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3194
3195 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003196 video_stream_encoder_->SetSource(&new_video_source,
3197 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003198 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003199 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003200 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003201
3202 stats = stats_proxy_->GetStats();
3203 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003204 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003205 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3206
3207 // Try to trigger overuse. Should not succeed.
3208 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003209 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003210 stats_proxy_->ResetMockStats();
3211
3212 stats = stats_proxy_->GetStats();
3213 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003214 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003215 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3216
3217 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003218 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003219 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003220 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003221 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003222 stats = stats_proxy_->GetStats();
3223 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003224 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003225 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003226
3227 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003228 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003229 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003230 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003231 stats = stats_proxy_->GetStats();
3232 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003233 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003234 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3235
3236 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003237 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003238 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003239 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003240 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003241 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003242 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003243 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003244 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003245 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003246 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3247
3248 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003249 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003250 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003251 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003252 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003253 stats = stats_proxy_->GetStats();
3254 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003255 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003256 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003257 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003258
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003260}
3261
mflodmancc3d4422017-08-03 08:27:51 -07003262TEST_F(VideoStreamEncoderTest,
3263 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003264 const int kWidth = 1280;
3265 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003267 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003268
asaperssonfab67072017-04-04 05:51:49 -07003269 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003270 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003271
asaperssonfab67072017-04-04 05:51:49 -07003272 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003273 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003274
asaperssonfab67072017-04-04 05:51:49 -07003275 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003276 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003277
asaperssonfab67072017-04-04 05:51:49 -07003278 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003279 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003280
kthelgason876222f2016-11-29 01:44:11 -08003281 // Expect a scale down.
3282 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003283 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003284
asapersson02465b82017-04-10 01:12:52 -07003285 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003286 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003287 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003288 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003289
asaperssonfab67072017-04-04 05:51:49 -07003290 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003291 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003292 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003293 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003294
asaperssonfab67072017-04-04 05:51:49 -07003295 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003296 EXPECT_EQ(std::numeric_limits<int>::max(),
3297 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003298
asaperssonfab67072017-04-04 05:51:49 -07003299 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003301 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003302 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003303
asapersson02465b82017-04-10 01:12:52 -07003304 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003305 EXPECT_EQ(std::numeric_limits<int>::max(),
3306 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003307
mflodmancc3d4422017-08-03 08:27:51 -07003308 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003309}
3310
mflodmancc3d4422017-08-03 08:27:51 -07003311TEST_F(VideoStreamEncoderTest,
3312 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003313 const int kWidth = 1280;
3314 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003316 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003317
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003318 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003319 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003320 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003321 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003322
3323 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003324 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003325 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003326 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3327 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3328
3329 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003330 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003331 EXPECT_THAT(source.sink_wants(),
3332 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003333 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3334 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3335 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3336
3337 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003338 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003339 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3340 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3341 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3342
mflodmancc3d4422017-08-03 08:27:51 -07003343 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003344}
3345
mflodmancc3d4422017-08-03 08:27:51 -07003346TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003347 const int kWidth = 1280;
3348 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003349 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003350 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003351
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003352 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003353 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003354 video_stream_encoder_->SetSource(&source,
3355 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003356 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3357 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003358 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003359
3360 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003361 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003362 EXPECT_THAT(source.sink_wants(),
3363 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003364 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3365 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3366 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3367
3368 // Trigger adapt down for same input resolution, expect no change.
3369 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3370 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003371 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003372 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3373 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3374 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3375
3376 // Trigger adapt down for larger input resolution, expect no change.
3377 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3378 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003380 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3381 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3382 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3383
mflodmancc3d4422017-08-03 08:27:51 -07003384 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003385}
3386
mflodmancc3d4422017-08-03 08:27:51 -07003387TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003388 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3389 const int kWidth = 640;
3390 const int kHeight = 360;
3391 const int64_t kFrameIntervalMs = 150;
3392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003393 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003394
3395 // Enable BALANCED preference, no initial limitation.
3396 AdaptingFrameForwarder source(&time_controller_);
3397 source.set_adaptation_enabled(true);
3398 video_stream_encoder_->SetSource(&source,
3399 webrtc::DegradationPreference::BALANCED);
3400
3401 int64_t timestamp_ms = kFrameIntervalMs;
3402 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3403 sink_.WaitForEncodedFrame(kWidth, kHeight);
3404 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3405 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3407 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3408
3409 // Trigger adapt down, expect reduced fps (640x360@15fps).
3410 video_stream_encoder_->TriggerQualityLow();
3411 timestamp_ms += kFrameIntervalMs;
3412 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3413 sink_.WaitForEncodedFrame(timestamp_ms);
3414 EXPECT_THAT(source.sink_wants(),
3415 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3416 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3417 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3418 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3419
3420 // Source requests 270p, expect reduced resolution (480x270@15fps).
3421 source.OnOutputFormatRequest(480, 270);
3422 timestamp_ms += kFrameIntervalMs;
3423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3424 WaitForEncodedFrame(480, 270);
3425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426
3427 // Trigger adapt down, expect reduced fps (480x270@10fps).
3428 video_stream_encoder_->TriggerQualityLow();
3429 timestamp_ms += kFrameIntervalMs;
3430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3431 sink_.WaitForEncodedFrame(timestamp_ms);
3432 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3435 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
3437 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3438 source.OnOutputFormatRequest(320, 180);
3439 timestamp_ms += kFrameIntervalMs;
3440 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3441 WaitForEncodedFrame(320, 180);
3442 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3443
3444 // Trigger adapt down, expect reduced fps (320x180@7fps).
3445 video_stream_encoder_->TriggerQualityLow();
3446 timestamp_ms += kFrameIntervalMs;
3447 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3448 sink_.WaitForEncodedFrame(timestamp_ms);
3449 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3450 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3451 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3452 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3453
3454 // Source requests VGA, expect increased resolution (640x360@7fps).
3455 source.OnOutputFormatRequest(640, 360);
3456 timestamp_ms += kFrameIntervalMs;
3457 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3458 WaitForEncodedFrame(timestamp_ms);
3459 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3462 video_stream_encoder_->TriggerQualityHigh();
3463 timestamp_ms += kFrameIntervalMs;
3464 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3465 WaitForEncodedFrame(timestamp_ms);
3466 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3468 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3469 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3470
3471 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3472 video_stream_encoder_->TriggerQualityHigh();
3473 timestamp_ms += kFrameIntervalMs;
3474 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3475 WaitForEncodedFrame(timestamp_ms);
3476 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3479 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3480
3481 // Trigger adapt up, expect increased fps (640x360@maxfps).
3482 video_stream_encoder_->TriggerQualityHigh();
3483 timestamp_ms += kFrameIntervalMs;
3484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3485 WaitForEncodedFrame(timestamp_ms);
3486 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3488 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3489 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3490
3491 video_stream_encoder_->Stop();
3492}
3493
3494TEST_F(VideoStreamEncoderTest,
3495 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3496 const int kWidth = 1280;
3497 const int kHeight = 720;
3498 const int64_t kFrameIntervalMs = 150;
3499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003500 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003501
3502 // Enable BALANCED preference, no initial limitation.
3503 AdaptingFrameForwarder source(&time_controller_);
3504 source.set_adaptation_enabled(true);
3505 video_stream_encoder_->SetSource(&source,
3506 webrtc::DegradationPreference::BALANCED);
3507
3508 int64_t timestamp_ms = kFrameIntervalMs;
3509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3510 sink_.WaitForEncodedFrame(kWidth, kHeight);
3511 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3512 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3514 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3515
3516 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3517 video_stream_encoder_->TriggerQualityLow();
3518 timestamp_ms += kFrameIntervalMs;
3519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3520 sink_.WaitForEncodedFrame(timestamp_ms);
3521 EXPECT_THAT(source.sink_wants(),
3522 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3525 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3526
3527 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3528 video_stream_encoder_->TriggerQualityLow();
3529 timestamp_ms += kFrameIntervalMs;
3530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3531 sink_.WaitForEncodedFrame(timestamp_ms);
3532 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3535 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3536
3537 // Trigger adapt down, expect reduced fps (640x360@15fps).
3538 video_stream_encoder_->TriggerQualityLow();
3539 timestamp_ms += kFrameIntervalMs;
3540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3541 WaitForEncodedFrame(timestamp_ms);
3542 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3545 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3546
3547 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3548 source.OnOutputFormatRequest(320, 180);
3549 timestamp_ms += kFrameIntervalMs;
3550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3551 WaitForEncodedFrame(320, 180);
3552 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3553 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3554
3555 // Trigger adapt down, expect reduced fps (320x180@7fps).
3556 video_stream_encoder_->TriggerCpuOveruse();
3557 timestamp_ms += kFrameIntervalMs;
3558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3559 WaitForEncodedFrame(timestamp_ms);
3560 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3563 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3564 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3565 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3566 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3567
3568 // Source requests HD, expect increased resolution (640x360@7fps).
3569 source.OnOutputFormatRequest(1280, 720);
3570 timestamp_ms += kFrameIntervalMs;
3571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3572 WaitForEncodedFrame(timestamp_ms);
3573 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3574 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3575
3576 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3577 video_stream_encoder_->TriggerCpuUnderuse();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3584 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3585 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588
3589 // Trigger adapt up, expect increased fps (640x360@maxfps).
3590 video_stream_encoder_->TriggerQualityHigh();
3591 video_stream_encoder_->TriggerCpuUnderuse();
3592 timestamp_ms += kFrameIntervalMs;
3593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3594 WaitForEncodedFrame(timestamp_ms);
3595 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3598 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3600 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3601 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3602
3603 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3604 video_stream_encoder_->TriggerQualityHigh();
3605 video_stream_encoder_->TriggerCpuUnderuse();
3606 timestamp_ms += kFrameIntervalMs;
3607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3608 WaitForEncodedFrame(timestamp_ms);
3609 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3611 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3612 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3613 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3614 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3616
3617 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3618 video_stream_encoder_->TriggerQualityHigh();
3619 video_stream_encoder_->TriggerCpuUnderuse();
3620 timestamp_ms += kFrameIntervalMs;
3621 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3622 WaitForEncodedFrame(timestamp_ms);
3623 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3625 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3627 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3628 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3629 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3630
3631 video_stream_encoder_->Stop();
3632}
3633
3634TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003635 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003636 const int kWidth = 1280;
3637 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003638 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003639 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003640
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003641 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003642 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003643 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003644 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003645
3646 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003647 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003648 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003649 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3650 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3651
3652 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003653 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003654 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003655 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3656 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3657
mflodmancc3d4422017-08-03 08:27:51 -07003658 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003659}
3660
mflodmancc3d4422017-08-03 08:27:51 -07003661TEST_F(VideoStreamEncoderTest,
3662 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003663 const int kWidth = 1280;
3664 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003666 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003667
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003668 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003669 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003670 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003671 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003672
3673 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003674 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003676 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003677 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3678
3679 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003680 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003681 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003682 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003683 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3684
mflodmancc3d4422017-08-03 08:27:51 -07003685 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003686}
3687
mflodmancc3d4422017-08-03 08:27:51 -07003688TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003689 const int kWidth = 1280;
3690 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003692 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003693
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003694 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003695 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003696 video_stream_encoder_->SetSource(&source,
3697 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003698
3699 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3700 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003701 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003702 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3703 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3704 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3705
3706 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003707 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003708 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3710 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3711 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3712
mflodmancc3d4422017-08-03 08:27:51 -07003713 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003714}
3715
mflodmancc3d4422017-08-03 08:27:51 -07003716TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003717 const int kWidth = 1280;
3718 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003719 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003720 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003721
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003722 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003723 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003724 video_stream_encoder_->SetSource(&source,
3725 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003726
3727 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3728 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3733
3734 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003735 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003736 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3738 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3739 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3740
mflodmancc3d4422017-08-03 08:27:51 -07003741 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003742}
3743
mflodmancc3d4422017-08-03 08:27:51 -07003744TEST_F(VideoStreamEncoderTest,
3745 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003746 const int kWidth = 1280;
3747 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003749 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003750
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003751 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003752 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003753 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003755 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003756
3757 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003758 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003759 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003760 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3761 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3762
3763 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003764 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003765 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003766 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003767 EXPECT_THAT(source.sink_wants(),
3768 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003769 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3770 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3771
3772 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003773 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003774 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003775 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3776 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3777 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3778
mflodmancc3d4422017-08-03 08:27:51 -07003779 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003780}
3781
mflodmancc3d4422017-08-03 08:27:51 -07003782TEST_F(VideoStreamEncoderTest,
3783 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003784 const int kWidth = 1280;
3785 const int kHeight = 720;
3786 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003787 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003788 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003789
3790 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3791 stats.input_frame_rate = kInputFps;
3792 stats_proxy_->SetMockStats(stats);
3793
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003794 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003795 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3796 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003797 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003798
3799 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003800 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003801 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3802 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003803 EXPECT_THAT(video_source_.sink_wants(),
3804 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003805
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003806 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003807 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003808 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003809 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003810 // Give the encoder queue time to process the change in degradation preference
3811 // by waiting for an encoded frame.
3812 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3813 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003814 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003815
3816 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003817 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003818 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3819 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003820 EXPECT_THAT(new_video_source.sink_wants(),
3821 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003822
3823 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003824 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003825 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003826
mflodmancc3d4422017-08-03 08:27:51 -07003827 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003828}
3829
mflodmancc3d4422017-08-03 08:27:51 -07003830TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003831 const int kWidth = 1280;
3832 const int kHeight = 720;
3833 const size_t kNumFrames = 10;
3834
Henrik Boström381d1092020-05-12 18:49:07 +02003835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003836 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003837
asaperssond0de2952017-04-21 01:47:31 -07003838 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003839 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003840 video_source_.set_adaptation_enabled(true);
3841
3842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3843 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3844
3845 int downscales = 0;
3846 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003847 video_source_.IncomingCapturedFrame(
3848 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3849 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003850
asaperssonfab67072017-04-04 05:51:49 -07003851 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003852 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003853 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003854 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003855
3856 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3857 ++downscales;
3858
3859 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3860 EXPECT_EQ(downscales,
3861 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3862 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003863 }
mflodmancc3d4422017-08-03 08:27:51 -07003864 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003865}
3866
mflodmancc3d4422017-08-03 08:27:51 -07003867TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003868 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3869 const int kWidth = 1280;
3870 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003872 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003873
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003874 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003875 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003876 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003878 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003879
Åsa Persson8c1bf952018-09-13 10:42:19 +02003880 int64_t timestamp_ms = kFrameIntervalMs;
3881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003882 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003883 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003884 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3885 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3886
3887 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003888 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003889 timestamp_ms += kFrameIntervalMs;
3890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3891 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003892 EXPECT_THAT(source.sink_wants(),
3893 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003894 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3896
3897 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003898 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003899 timestamp_ms += kFrameIntervalMs;
3900 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003901 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003902 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003903 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3904 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3905
3906 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003907 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003908 timestamp_ms += kFrameIntervalMs;
3909 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3910 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003911 EXPECT_THAT(source.sink_wants(),
3912 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003913 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3914 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3915
3916 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003917 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003918 timestamp_ms += kFrameIntervalMs;
3919 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003920 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003921 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3923 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3924
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003926}
3927
mflodmancc3d4422017-08-03 08:27:51 -07003928TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003929 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3930 const int kWidth = 1280;
3931 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003932 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003933 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003934
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003935 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003936 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003937 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003938 video_stream_encoder_->SetSource(&source,
3939 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003940
Åsa Persson8c1bf952018-09-13 10:42:19 +02003941 int64_t timestamp_ms = kFrameIntervalMs;
3942 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003943 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003944 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003945 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3946 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3947
3948 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003949 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003950 timestamp_ms += kFrameIntervalMs;
3951 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3952 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003953 EXPECT_THAT(source.sink_wants(),
3954 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003955 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3956 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3957
3958 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003959 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003960 timestamp_ms += kFrameIntervalMs;
3961 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003962 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003963 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003964 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3965 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3966
3967 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003968 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003969 timestamp_ms += kFrameIntervalMs;
3970 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3971 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003972 EXPECT_THAT(source.sink_wants(),
3973 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003974 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3975 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3976
3977 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003978 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003979 timestamp_ms += kFrameIntervalMs;
3980 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003981 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003982 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003983 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3984 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3985
mflodmancc3d4422017-08-03 08:27:51 -07003986 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003987}
3988
Sergey Silkin41c650b2019-10-14 13:12:19 +02003989TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3990 fake_encoder_.SetResolutionBitrateLimits(
3991 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3992
Henrik Boström381d1092020-05-12 18:49:07 +02003993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003994 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3995 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3996 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3997 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003998
3999 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004000 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004001 source.set_adaptation_enabled(true);
4002 video_stream_encoder_->SetSource(
4003 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4004
4005 // Insert 720p frame.
4006 int64_t timestamp_ms = kFrameIntervalMs;
4007 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4008 WaitForEncodedFrame(1280, 720);
4009
4010 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004011 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004012 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4013 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4014 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4015 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004016 video_stream_encoder_->TriggerQualityLow();
4017
4018 // Insert 720p frame. It should be downscaled and encoded.
4019 timestamp_ms += kFrameIntervalMs;
4020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4021 WaitForEncodedFrame(960, 540);
4022
4023 // Trigger adapt up. Higher resolution should not be requested duo to lack
4024 // of bitrate.
4025 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004026 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004027
4028 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004030 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4031 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4032 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4033 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004034
4035 // Trigger adapt up. Higher resolution should be requested.
4036 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004037 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004038
4039 video_stream_encoder_->Stop();
4040}
4041
4042TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4043 fake_encoder_.SetResolutionBitrateLimits(
4044 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4045
4046 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004048 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4049 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4050 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4051 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004052
4053 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004054 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004055 source.set_adaptation_enabled(true);
4056 video_stream_encoder_->SetSource(
4057 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4058
4059 // Insert 720p frame. It should be dropped and lower resolution should be
4060 // requested.
4061 int64_t timestamp_ms = kFrameIntervalMs;
4062 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4063 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004064 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004065
4066 // Insert 720p frame. It should be downscaled and encoded.
4067 timestamp_ms += kFrameIntervalMs;
4068 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4069 WaitForEncodedFrame(960, 540);
4070
4071 video_stream_encoder_->Stop();
4072}
4073
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004074class BalancedDegradationTest : public VideoStreamEncoderTest {
4075 protected:
4076 void SetupTest() {
4077 // Reset encoder for field trials to take effect.
4078 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004079 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004080
4081 // Enable BALANCED preference.
4082 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004083 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4084 }
4085
Asa Persson606d3cb2021-10-04 10:07:11 +02004086 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004088 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004089 }
4090
Åsa Persson45b176f2019-09-30 11:19:05 +02004091 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004092 timestamp_ms_ += kFrameIntervalMs;
4093 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004094 }
4095
4096 void InsertFrameAndWaitForEncoded() {
4097 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004098 sink_.WaitForEncodedFrame(timestamp_ms_);
4099 }
4100
4101 const int kWidth = 640; // pixels:640x360=230400
4102 const int kHeight = 360;
4103 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4104 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004105 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004106};
4107
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004108TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004109 test::ScopedFieldTrials field_trials(
4110 "WebRTC-Video-BalancedDegradationSettings/"
4111 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4112 SetupTest();
4113
4114 // Force input frame rate.
4115 const int kInputFps = 24;
4116 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4117 stats.input_frame_rate = kInputFps;
4118 stats_proxy_->SetMockStats(stats);
4119
Åsa Persson45b176f2019-09-30 11:19:05 +02004120 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004121 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004122
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004123 // Trigger adapt down, expect scaled down framerate and resolution,
4124 // since Fps diff (input-requested:0) < threshold.
4125 video_stream_encoder_->TriggerQualityLow();
4126 EXPECT_THAT(source_.sink_wants(),
4127 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004128
4129 video_stream_encoder_->Stop();
4130}
4131
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004132TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004133 test::ScopedFieldTrials field_trials(
4134 "WebRTC-Video-BalancedDegradationSettings/"
4135 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4136 SetupTest();
4137
4138 // Force input frame rate.
4139 const int kInputFps = 25;
4140 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4141 stats.input_frame_rate = kInputFps;
4142 stats_proxy_->SetMockStats(stats);
4143
Åsa Persson45b176f2019-09-30 11:19:05 +02004144 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004145 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004146
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004147 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4148 // Fps diff (input-requested:1) == threshold.
4149 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004150 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004151
4152 video_stream_encoder_->Stop();
4153}
4154
4155TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4156 test::ScopedFieldTrials field_trials(
4157 "WebRTC-Video-BalancedDegradationSettings/"
4158 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4159 SetupTest();
4160
4161 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4162
Åsa Persson45b176f2019-09-30 11:19:05 +02004163 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004164 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004165
4166 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4167 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004168 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004169
4170 video_stream_encoder_->Stop();
4171}
4172
Åsa Perssonccfb3402019-09-25 15:13:04 +02004173TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004174 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004175 "WebRTC-Video-BalancedDegradationSettings/"
4176 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004177 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004178
Asa Persson606d3cb2021-10-04 10:07:11 +02004179 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4180 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4181 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004182
Åsa Persson45b176f2019-09-30 11:19:05 +02004183 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004184 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004185 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4186
4187 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4188 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004189 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004190 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004191 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4192
4193 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4194 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004195 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004196 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004197 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4198
Åsa Persson30ab0152019-08-27 12:22:33 +02004199 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4200 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004201 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004202 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004203 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004204 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4205
4206 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004207 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004208 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004209 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004210
Åsa Persson30ab0152019-08-27 12:22:33 +02004211 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004212 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004213 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004214 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004215 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004216 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4217
4218 video_stream_encoder_->Stop();
4219}
4220
Åsa Perssonccfb3402019-09-25 15:13:04 +02004221TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004222 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4223 test::ScopedFieldTrials field_trials(
4224 "WebRTC-Video-BalancedDegradationSettings/"
4225 "pixels:57600|129600|230400,fps:7|24|24/");
4226 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004227 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004228
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004229 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004230
4231 // Insert frame, expect scaled down:
4232 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4233 InsertFrame();
4234 EXPECT_FALSE(WaitForFrame(1000));
4235 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4236 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4237
4238 // Insert frame, expect scaled down:
4239 // resolution (320x180@24fps).
4240 InsertFrame();
4241 EXPECT_FALSE(WaitForFrame(1000));
4242 EXPECT_LT(source_.sink_wants().max_pixel_count,
4243 source_.last_wants().max_pixel_count);
4244 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4245
4246 // Frame should not be dropped (min pixels per frame reached).
4247 InsertFrameAndWaitForEncoded();
4248
4249 video_stream_encoder_->Stop();
4250}
4251
4252TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004253 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004254 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004255 "WebRTC-Video-BalancedDegradationSettings/"
4256 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004257 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004258
Asa Persson606d3cb2021-10-04 10:07:11 +02004259 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4260 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4261 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004262
Åsa Persson45b176f2019-09-30 11:19:05 +02004263 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004264 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004265 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4266
4267 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4268 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004269 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004270 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004271 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4272
4273 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4274 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004275 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004276 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004277 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4278
4279 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4280 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004281 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004282 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004283 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4284
Åsa Persson30ab0152019-08-27 12:22:33 +02004285 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4286 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004287 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004288 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004289 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4290
4291 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4292 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004293 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004294 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4295
4296 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004297 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004298 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004299 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004300 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004301 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4302
4303 video_stream_encoder_->Stop();
4304}
4305
Åsa Perssonccfb3402019-09-25 15:13:04 +02004306TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004308 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004309 "WebRTC-Video-BalancedDegradationSettings/"
4310 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004311 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004312
Asa Persson606d3cb2021-10-04 10:07:11 +02004313 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4314 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4315 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4316 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4317 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004318
Åsa Persson45b176f2019-09-30 11:19:05 +02004319 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004320 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004321 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4322
4323 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4324 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004325 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004326 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004327 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4328
4329 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4330 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004331 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004332 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004333 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4334
4335 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4336 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004337 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004338 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004339 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4340
4341 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4342 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004343 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004347 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004348 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004349 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004350 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004351 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4352
4353 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004354 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004356 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004357 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4358
4359 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004360 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004361 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004362 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004363 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004364 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
Åsa Persson1b247f12019-08-14 17:26:39 +02004366 video_stream_encoder_->Stop();
4367}
4368
mflodmancc3d4422017-08-03 08:27:51 -07004369TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004370 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4371 const int kWidth = 1280;
4372 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004374 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004375
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004376 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004377 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004378 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004379 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004380 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004381
Åsa Persson8c1bf952018-09-13 10:42:19 +02004382 int64_t timestamp_ms = kFrameIntervalMs;
4383 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004384 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004385 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004386 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4387 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4388 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4389 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
4391 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004392 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004393 timestamp_ms += kFrameIntervalMs;
4394 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4395 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004396 EXPECT_THAT(source.sink_wants(),
4397 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004398 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4399 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4400 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4401 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4402
4403 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004404 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004405 timestamp_ms += kFrameIntervalMs;
4406 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4407 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004408 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004409 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4410 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4411 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4412 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4413
Jonathan Yubc771b72017-12-08 17:04:29 -08004414 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004415 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004416 timestamp_ms += kFrameIntervalMs;
4417 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4418 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004420 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004422 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004423 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4424
Jonathan Yubc771b72017-12-08 17:04:29 -08004425 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004426 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004427 timestamp_ms += kFrameIntervalMs;
4428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4429 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004430 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004431 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004432 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4434 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4435 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4436
Jonathan Yubc771b72017-12-08 17:04:29 -08004437 // Trigger quality adapt down, expect no change (min resolution reached).
4438 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004439 timestamp_ms += kFrameIntervalMs;
4440 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4441 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004442 EXPECT_THAT(source.sink_wants(), FpsMax());
4443 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004444 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4445 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4446 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4447 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4448
Evan Shrubsole64469032020-06-11 10:45:29 +02004449 // Trigger quality adapt up, expect upscaled resolution (480x270).
4450 video_stream_encoder_->TriggerQualityHigh();
4451 timestamp_ms += kFrameIntervalMs;
4452 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4453 WaitForEncodedFrame(timestamp_ms);
4454 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4455 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4456 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4457 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4458 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4459
4460 // Trigger quality and cpu adapt up since both are most limited, expect
4461 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004462 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004463 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4466 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004467 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4470 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004471 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004472
Evan Shrubsole64469032020-06-11 10:45:29 +02004473 // Trigger quality and cpu adapt up since both are most limited, expect
4474 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004475 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004476 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004477 timestamp_ms += kFrameIntervalMs;
4478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4479 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004480 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004481 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004482 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004484 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4485 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004486
Evan Shrubsole64469032020-06-11 10:45:29 +02004487 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4488 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004489 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004490 timestamp_ms += kFrameIntervalMs;
4491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4492 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004493 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004494 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004496 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004497 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004498
4499 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004500 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004501 timestamp_ms += kFrameIntervalMs;
4502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004503 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004504 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004505 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004506 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4507 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004508 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004509 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004510
mflodmancc3d4422017-08-03 08:27:51 -07004511 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004512}
4513
mflodmancc3d4422017-08-03 08:27:51 -07004514TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004515 const int kWidth = 640;
4516 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004517
Henrik Boström381d1092020-05-12 18:49:07 +02004518 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004519 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004520
perkj803d97f2016-11-01 11:45:46 -07004521 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004522 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004523 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004524 }
4525
mflodmancc3d4422017-08-03 08:27:51 -07004526 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004527 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004528 video_source_.IncomingCapturedFrame(CreateFrame(
4529 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004530 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004531 }
4532
mflodmancc3d4422017-08-03 08:27:51 -07004533 video_stream_encoder_->Stop();
4534 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004535 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004536
Ying Wangef3998f2019-12-09 13:06:53 +01004537 EXPECT_METRIC_EQ(
4538 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4539 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004540 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4541}
4542
mflodmancc3d4422017-08-03 08:27:51 -07004543TEST_F(VideoStreamEncoderTest,
4544 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004546 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004547 const int kWidth = 640;
4548 const int kHeight = 360;
4549
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004550 video_stream_encoder_->SetSource(&video_source_,
4551 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004552
4553 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4554 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004555 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004556 }
4557
mflodmancc3d4422017-08-03 08:27:51 -07004558 video_stream_encoder_->Stop();
4559 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004560 stats_proxy_.reset();
4561
4562 EXPECT_EQ(0,
4563 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4564}
4565
Per Kjellanderdcef6412020-10-07 15:09:05 +02004566TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4567 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004568 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004569 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004570
4571 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004572 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004573 SimulcastRateAllocator(fake_encoder_.config())
4574 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004575 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004576
Henrik Boström381d1092020-05-12 18:49:07 +02004577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004578 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004579
sprang57c2fff2017-01-16 06:24:02 -08004580 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004581 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4582 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004583 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4584 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4585
Erik Språngd7329ca2019-02-21 21:19:53 +01004586 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004587 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004588 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004589
Per Kjellanderdcef6412020-10-07 15:09:05 +02004590 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004591 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004592 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4593 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004594 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004595 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004596
Per Kjellanderdcef6412020-10-07 15:09:05 +02004597 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004598 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004599 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004600 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004601 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4602 WaitForEncodedFrame(CurrentTimeMs());
4603 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004604 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004605 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004606
mflodmancc3d4422017-08-03 08:27:51 -07004607 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004608}
4609
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004610TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004611 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004612 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004613 kVideoLayersAllocation);
4614
4615 const int kDefaultFps = 30;
4616
4617 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004618 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004619
4620 video_source_.IncomingCapturedFrame(
4621 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4622 WaitForEncodedFrame(CurrentTimeMs());
4623 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4624 VideoLayersAllocation last_layer_allocation =
4625 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004626 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004627 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4628
4629 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004630 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004631 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004632 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004633 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4634
Erik Språng9d69cbe2020-10-22 17:44:42 +02004635 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004636 int number_of_layers_allocation = 1;
4637 const int64_t start_time_ms = CurrentTimeMs();
4638 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4639 video_source_.IncomingCapturedFrame(
4640 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4641 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004642 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4643 number_of_layers_allocation = sink_.number_of_layers_allocations();
4644 VideoLayersAllocation new_allocation =
4645 sink_.GetLastVideoLayersAllocation();
4646 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4647 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4648 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4649 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4650 .target_bitrate_per_temporal_layer,
4651 last_layer_allocation.active_spatial_layers[0]
4652 .target_bitrate_per_temporal_layer);
4653 last_layer_allocation = new_allocation;
4654 }
4655 }
4656 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4657 video_stream_encoder_->Stop();
4658}
4659
4660TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004661 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004662 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4663 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4664 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004665 VideoEncoderConfig video_encoder_config;
4666 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4667 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004668 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004669 video_encoder_config.content_type =
4670 VideoEncoderConfig::ContentType::kRealtimeVideo;
4671 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004672 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004673 VideoEncoder::GetDefaultVp8Settings());
4674 for (auto& layer : video_encoder_config.simulcast_layers) {
4675 layer.num_temporal_layers = 2;
4676 }
4677 // Simulcast layers are used for enabling/disabling streams.
4678 video_encoder_config.simulcast_layers[0].active = true;
4679 video_encoder_config.simulcast_layers[1].active = false;
4680 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004681 ConfigureEncoder(std::move(video_encoder_config),
4682 VideoStreamEncoder::BitrateAllocationCallbackType::
4683 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004684
4685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004686 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004687
4688 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4689 WaitForEncodedFrame(CurrentTimeMs());
4690 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4691 VideoLayersAllocation last_layer_allocation =
4692 sink_.GetLastVideoLayersAllocation();
4693
4694 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4695 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4696 .target_bitrate_per_temporal_layer,
4697 SizeIs(2));
4698 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4699 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4700 video_stream_encoder_->Stop();
4701}
4702
4703TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004704 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004705 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4706 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4707 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004708 VideoEncoderConfig video_encoder_config;
4709 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4710 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004711 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004712 video_encoder_config.content_type =
4713 VideoEncoderConfig::ContentType::kRealtimeVideo;
4714 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004715 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004716 VideoEncoder::GetDefaultVp8Settings());
4717 for (auto& layer : video_encoder_config.simulcast_layers) {
4718 layer.num_temporal_layers = 2;
4719 }
4720 // Simulcast layers are used for enabling/disabling streams.
4721 video_encoder_config.simulcast_layers[0].active = true;
4722 video_encoder_config.simulcast_layers[1].active = false;
4723 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004724 ConfigureEncoder(std::move(video_encoder_config),
4725 VideoStreamEncoder::BitrateAllocationCallbackType::
4726 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004727
4728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004729 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004730
4731 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4732 WaitForEncodedFrame(CurrentTimeMs());
4733 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4734 VideoLayersAllocation last_layer_allocation =
4735 sink_.GetLastVideoLayersAllocation();
4736
4737 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4738 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4739 .target_bitrate_per_temporal_layer,
4740 SizeIs(2));
4741 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4742
4743 video_stream_encoder_->Stop();
4744}
4745
4746TEST_F(VideoStreamEncoderTest,
4747 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4748 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4749 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004750 VideoEncoderConfig video_encoder_config;
4751 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4752 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004753 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004754 video_encoder_config.content_type =
4755 VideoEncoderConfig::ContentType::kRealtimeVideo;
4756 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4757 vp9_settings.numberOfSpatialLayers = 2;
4758 vp9_settings.numberOfTemporalLayers = 2;
4759 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4760 vp9_settings.automaticResizeOn = false;
4761 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004762 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004763 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004764 ConfigureEncoder(std::move(video_encoder_config),
4765 VideoStreamEncoder::BitrateAllocationCallbackType::
4766 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004767
4768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004769 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004770
4771 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4772 WaitForEncodedFrame(CurrentTimeMs());
4773 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4774 VideoLayersAllocation last_layer_allocation =
4775 sink_.GetLastVideoLayersAllocation();
4776
4777 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4778 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4779 .target_bitrate_per_temporal_layer,
4780 SizeIs(2));
4781 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4782 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4783 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4784 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4785 .target_bitrate_per_temporal_layer,
4786 SizeIs(2));
4787 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4788 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4789 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4790
4791 // Since full SVC is used, expect the top layer to utilize the full target
4792 // rate.
4793 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4794 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004795 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004796 video_stream_encoder_->Stop();
4797}
4798
4799TEST_F(VideoStreamEncoderTest,
4800 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4801 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4802 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004803 VideoEncoderConfig video_encoder_config;
4804 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4805 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004806 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004807 video_encoder_config.content_type =
4808 VideoEncoderConfig::ContentType::kRealtimeVideo;
4809 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4810 vp9_settings.numberOfSpatialLayers = 2;
4811 vp9_settings.numberOfTemporalLayers = 2;
4812 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4813 vp9_settings.automaticResizeOn = false;
4814 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004815 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004816 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004817 ConfigureEncoder(std::move(video_encoder_config),
4818 VideoStreamEncoder::BitrateAllocationCallbackType::
4819 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004820
4821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004822 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823
4824 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4825 WaitForEncodedFrame(CurrentTimeMs());
4826 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4827 VideoLayersAllocation last_layer_allocation =
4828 sink_.GetLastVideoLayersAllocation();
4829
4830 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4831 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4832 .target_bitrate_per_temporal_layer,
4833 SizeIs(1));
4834 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4835 .target_bitrate_per_temporal_layer,
4836 SizeIs(1));
4837 // Since full SVC is used, expect the top layer to utilize the full target
4838 // rate.
4839 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4840 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004841 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004842 video_stream_encoder_->Stop();
4843}
4844
4845TEST_F(VideoStreamEncoderTest,
4846 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4847 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4848 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004849 VideoEncoderConfig video_encoder_config;
4850 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4851 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004852 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004853 video_encoder_config.content_type =
4854 VideoEncoderConfig::ContentType::kRealtimeVideo;
4855 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4856 vp9_settings.numberOfSpatialLayers = 2;
4857 vp9_settings.numberOfTemporalLayers = 2;
4858 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4859 vp9_settings.automaticResizeOn = false;
4860 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004861 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004862 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004863 ConfigureEncoder(std::move(video_encoder_config),
4864 VideoStreamEncoder::BitrateAllocationCallbackType::
4865 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004866
4867 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004868 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004869
4870 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4871 WaitForEncodedFrame(CurrentTimeMs());
4872 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4873 VideoLayersAllocation last_layer_allocation =
4874 sink_.GetLastVideoLayersAllocation();
4875
4876 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4877 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4878 .target_bitrate_per_temporal_layer,
4879 SizeIs(2));
4880 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4881 .target_bitrate_per_temporal_layer,
4882 SizeIs(2));
4883 // Since KSVC is, spatial layers are independend except on key frames.
4884 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4885 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004886 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004887 video_stream_encoder_->Stop();
4888}
4889
4890TEST_F(VideoStreamEncoderTest,
4891 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4892 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4893 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4894 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004895 VideoEncoderConfig video_encoder_config;
4896 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4897 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004898 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004899 video_encoder_config.content_type =
4900 VideoEncoderConfig::ContentType::kRealtimeVideo;
4901 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4902 vp9_settings.numberOfSpatialLayers = 3;
4903 vp9_settings.numberOfTemporalLayers = 2;
4904 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4905 vp9_settings.automaticResizeOn = false;
4906 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004907 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004908 vp9_settings);
4909 // Simulcast layers are used for enabling/disabling streams.
4910 video_encoder_config.simulcast_layers.resize(3);
4911 video_encoder_config.simulcast_layers[0].active = false;
4912 video_encoder_config.simulcast_layers[1].active = true;
4913 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004914 ConfigureEncoder(std::move(video_encoder_config),
4915 VideoStreamEncoder::BitrateAllocationCallbackType::
4916 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004917
4918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004919 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920
4921 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4922 WaitForEncodedFrame(CurrentTimeMs());
4923 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4924 VideoLayersAllocation last_layer_allocation =
4925 sink_.GetLastVideoLayersAllocation();
4926
4927 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4928 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4929 .target_bitrate_per_temporal_layer,
4930 SizeIs(2));
4931 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4932 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4933
4934 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4935 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4936 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4937 .target_bitrate_per_temporal_layer,
4938 SizeIs(2));
4939 // Since full SVC is used, expect the top layer to utilize the full target
4940 // rate.
4941 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4942 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004943 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004944 video_stream_encoder_->Stop();
4945}
4946
4947TEST_F(VideoStreamEncoderTest,
4948 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4949 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4950 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4951 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004952 VideoEncoderConfig video_encoder_config;
4953 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4954 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004955 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004956 video_encoder_config.content_type =
4957 VideoEncoderConfig::ContentType::kRealtimeVideo;
4958 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4959 vp9_settings.numberOfSpatialLayers = 3;
4960 vp9_settings.numberOfTemporalLayers = 2;
4961 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4962 vp9_settings.automaticResizeOn = false;
4963 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004964 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004965 vp9_settings);
4966 // Simulcast layers are used for enabling/disabling streams.
4967 video_encoder_config.simulcast_layers.resize(3);
4968 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004969 ConfigureEncoder(std::move(video_encoder_config),
4970 VideoStreamEncoder::BitrateAllocationCallbackType::
4971 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004972
4973 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004974 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004975
4976 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4977 WaitForEncodedFrame(CurrentTimeMs());
4978 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4979 VideoLayersAllocation last_layer_allocation =
4980 sink_.GetLastVideoLayersAllocation();
4981
4982 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4983 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4984 .target_bitrate_per_temporal_layer,
4985 SizeIs(2));
4986 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4987 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4988
4989 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4990 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4991 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4992 .target_bitrate_per_temporal_layer,
4993 SizeIs(2));
4994 video_stream_encoder_->Stop();
4995}
4996
4997TEST_F(VideoStreamEncoderTest,
4998 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4999 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5000 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5001 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005002 VideoEncoderConfig video_encoder_config;
5003 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5004 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005005 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005006 video_encoder_config.content_type =
5007 VideoEncoderConfig::ContentType::kRealtimeVideo;
5008 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5009 vp9_settings.numberOfSpatialLayers = 3;
5010 vp9_settings.numberOfTemporalLayers = 2;
5011 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5012 vp9_settings.automaticResizeOn = false;
5013 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005014 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005015 vp9_settings);
5016 // Simulcast layers are used for enabling/disabling streams.
5017 video_encoder_config.simulcast_layers.resize(3);
5018 video_encoder_config.simulcast_layers[0].active = false;
5019 video_encoder_config.simulcast_layers[1].active = false;
5020 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005021 ConfigureEncoder(std::move(video_encoder_config),
5022 VideoStreamEncoder::BitrateAllocationCallbackType::
5023 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005024
5025 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005026 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005027
5028 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5029 WaitForEncodedFrame(CurrentTimeMs());
5030 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5031 VideoLayersAllocation last_layer_allocation =
5032 sink_.GetLastVideoLayersAllocation();
5033
5034 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5035 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5036 .target_bitrate_per_temporal_layer,
5037 SizeIs(2));
5038 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5039 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5040 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5041 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005042 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005043 video_stream_encoder_->Stop();
5044}
5045
5046TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5047 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005048 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005049 kVideoLayersAllocation);
5050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005051 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005052
5053 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5054 WaitForEncodedFrame(CurrentTimeMs());
5055 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5056 VideoLayersAllocation last_layer_allocation =
5057 sink_.GetLastVideoLayersAllocation();
5058
5059 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5060 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5061 .target_bitrate_per_temporal_layer,
5062 SizeIs(1));
5063 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5064 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005065 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005066 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5067 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5068 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5069 video_stream_encoder_->Stop();
5070}
5071
5072TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005073 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5074 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005075 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005076 kVideoLayersAllocation);
5077
5078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005079 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005080
5081 video_source_.IncomingCapturedFrame(
5082 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5083 WaitForEncodedFrame(CurrentTimeMs());
5084 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5085 VideoLayersAllocation last_layer_allocation =
5086 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005087 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005088 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5089 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5090 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005091 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005092
5093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005094 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5095 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005096 video_source_.IncomingCapturedFrame(
5097 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5098 WaitForEncodedFrame(CurrentTimeMs());
5099
5100 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5101 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5102 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5103 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5104 .target_bitrate_per_temporal_layer[0],
5105 DataRate::Zero());
5106
5107 video_stream_encoder_->Stop();
5108}
5109
Per Kjellander4190ce92020-12-15 17:24:55 +01005110TEST_F(VideoStreamEncoderTest,
5111 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5112 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005113 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005114 kVideoLayersAllocation);
5115
5116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005117 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5118 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005119
5120 video_source_.IncomingCapturedFrame(
5121 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5122 WaitForEncodedFrame(CurrentTimeMs());
5123 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5124 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5125 SizeIs(2));
5126 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5127 codec_width_);
5128 EXPECT_EQ(
5129 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5130 codec_height_);
5131
5132 video_source_.IncomingCapturedFrame(
5133 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5134 WaitForEncodedFrame(CurrentTimeMs());
5135 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5136 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5137 SizeIs(2));
5138 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5139 codec_width_ / 2);
5140 EXPECT_EQ(
5141 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5142 codec_height_ / 2);
5143
5144 video_stream_encoder_->Stop();
5145}
5146
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005147TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5148 // 2 TLs configured, temporal layers supported by encoder.
5149 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005150 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005151 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005152 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005153 fake_encoder_.SetTemporalLayersSupported(0, true);
5154
5155 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005156 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005157 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005158 kNumTemporalLayers, /*temporal_id*/ 0,
5159 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005160 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005161 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005162 kNumTemporalLayers, /*temporal_id*/ 1,
5163 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005164 VideoBitrateAllocation expected_bitrate;
5165 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5166 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5167
5168 VerifyAllocatedBitrate(expected_bitrate);
5169 video_stream_encoder_->Stop();
5170}
5171
5172TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5173 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005174 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005175 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005176 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005177 fake_encoder_.SetTemporalLayersSupported(0, false);
5178
5179 // Temporal layers not supported by the encoder.
5180 // Total bitrate should be at ti:0.
5181 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005182 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005183
5184 VerifyAllocatedBitrate(expected_bitrate);
5185 video_stream_encoder_->Stop();
5186}
5187
5188TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005189 webrtc::test::ScopedFieldTrials field_trials(
5190 "WebRTC-Video-QualityScalerSettings/"
5191 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5192 // Reset encoder for field trials to take effect.
5193 ConfigureEncoder(video_encoder_config_.Copy());
5194
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005195 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005196 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005197 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005198 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005199 fake_encoder_.SetTemporalLayersSupported(0, true);
5200 fake_encoder_.SetTemporalLayersSupported(1, false);
5201
5202 const int kS0Bps = 150000;
5203 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005204 kS0Bps *
5205 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5206 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005207 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005208 kS0Bps *
5209 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5210 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005211 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005212 // Temporal layers not supported by si:1.
5213 VideoBitrateAllocation expected_bitrate;
5214 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5215 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5216 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5217
5218 VerifyAllocatedBitrate(expected_bitrate);
5219 video_stream_encoder_->Stop();
5220}
5221
Niels Möller7dc26b72017-12-06 10:27:48 +01005222TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5223 const int kFrameWidth = 1280;
5224 const int kFrameHeight = 720;
5225 const int kFramerate = 24;
5226
Henrik Boström381d1092020-05-12 18:49:07 +02005227 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005228 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005229 test::FrameForwarder source;
5230 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005231 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005232
5233 // Insert a single frame, triggering initial configuration.
5234 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5235 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5236
5237 EXPECT_EQ(
5238 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5239 kDefaultFramerate);
5240
5241 // Trigger reconfigure encoder (without resetting the entire instance).
5242 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005243 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5244 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005245 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005246 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005247 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005248 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5249
5250 // Detector should be updated with fps limit from codec config.
5251 EXPECT_EQ(
5252 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5253 kFramerate);
5254
5255 // Trigger overuse, max framerate should be reduced.
5256 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5257 stats.input_frame_rate = kFramerate;
5258 stats_proxy_->SetMockStats(stats);
5259 video_stream_encoder_->TriggerCpuOveruse();
5260 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5261 int adapted_framerate =
5262 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5263 EXPECT_LT(adapted_framerate, kFramerate);
5264
5265 // Trigger underuse, max framerate should go back to codec configured fps.
5266 // Set extra low fps, to make sure it's actually reset, not just incremented.
5267 stats = stats_proxy_->GetStats();
5268 stats.input_frame_rate = adapted_framerate / 2;
5269 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005270 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005271 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5272 EXPECT_EQ(
5273 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5274 kFramerate);
5275
5276 video_stream_encoder_->Stop();
5277}
5278
5279TEST_F(VideoStreamEncoderTest,
5280 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5281 const int kFrameWidth = 1280;
5282 const int kFrameHeight = 720;
5283 const int kLowFramerate = 15;
5284 const int kHighFramerate = 25;
5285
Henrik Boström381d1092020-05-12 18:49:07 +02005286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005288 test::FrameForwarder source;
5289 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005290 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005291
5292 // Trigger initial configuration.
5293 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005294 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5295 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005296 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005297 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005298 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005299 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005300 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5301
5302 EXPECT_EQ(
5303 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5304 kLowFramerate);
5305
5306 // Trigger overuse, max framerate should be reduced.
5307 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5308 stats.input_frame_rate = kLowFramerate;
5309 stats_proxy_->SetMockStats(stats);
5310 video_stream_encoder_->TriggerCpuOveruse();
5311 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5312 int adapted_framerate =
5313 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5314 EXPECT_LT(adapted_framerate, kLowFramerate);
5315
5316 // Reconfigure the encoder with a new (higher max framerate), max fps should
5317 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005318 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005319 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5320 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005321 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005322 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5323
5324 EXPECT_EQ(
5325 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5326 adapted_framerate);
5327
5328 // Trigger underuse, max framerate should go back to codec configured fps.
5329 stats = stats_proxy_->GetStats();
5330 stats.input_frame_rate = adapted_framerate;
5331 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005332 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005333 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5334 EXPECT_EQ(
5335 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5336 kHighFramerate);
5337
5338 video_stream_encoder_->Stop();
5339}
5340
mflodmancc3d4422017-08-03 08:27:51 -07005341TEST_F(VideoStreamEncoderTest,
5342 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005343 const int kFrameWidth = 1280;
5344 const int kFrameHeight = 720;
5345 const int kFramerate = 24;
5346
Henrik Boström381d1092020-05-12 18:49:07 +02005347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005349 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005350 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005351 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005352
5353 // Trigger initial configuration.
5354 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005355 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5356 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005357 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005358 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005359 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005360 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005361 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005362
Niels Möller7dc26b72017-12-06 10:27:48 +01005363 EXPECT_EQ(
5364 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5365 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005366
5367 // Trigger overuse, max framerate should be reduced.
5368 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5369 stats.input_frame_rate = kFramerate;
5370 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005371 video_stream_encoder_->TriggerCpuOveruse();
5372 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005373 int adapted_framerate =
5374 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005375 EXPECT_LT(adapted_framerate, kFramerate);
5376
5377 // Change degradation preference to not enable framerate scaling. Target
5378 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005379 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005380 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005381 EXPECT_EQ(
5382 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5383 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005384
mflodmancc3d4422017-08-03 08:27:51 -07005385 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005386}
5387
mflodmancc3d4422017-08-03 08:27:51 -07005388TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005389 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005391 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5392 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5393 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005394 const int kWidth = 640;
5395 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005396
asaperssonfab67072017-04-04 05:51:49 -07005397 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005398
5399 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005400 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005401
5402 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005403 EXPECT_TRUE_WAIT(
5404 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005405
sprangc5d62e22017-04-02 23:53:04 -07005406 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005407
asaperssonfab67072017-04-04 05:51:49 -07005408 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005409 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005410 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005411
5412 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005413 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005414
Henrik Boström2671dac2020-05-19 16:29:09 +02005415 EXPECT_TRUE_WAIT(
5416 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005417
mflodmancc3d4422017-08-03 08:27:51 -07005418 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005419}
5420
mflodmancc3d4422017-08-03 08:27:51 -07005421TEST_F(VideoStreamEncoderTest,
5422 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005423 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005425 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5426 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5427 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005428 const int kWidth = 640;
5429 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005430
5431 // We expect the n initial frames to get dropped.
5432 int i;
5433 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005434 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005435 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005436 }
5437 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005438 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005439 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005440
5441 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005442 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005443
mflodmancc3d4422017-08-03 08:27:51 -07005444 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005445}
5446
mflodmancc3d4422017-08-03 08:27:51 -07005447TEST_F(VideoStreamEncoderTest,
5448 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005449 const int kWidth = 640;
5450 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005452 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005453
5454 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005455 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005456 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005457
asaperssonfab67072017-04-04 05:51:49 -07005458 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005459 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005460 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005461
mflodmancc3d4422017-08-03 08:27:51 -07005462 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005463}
5464
mflodmancc3d4422017-08-03 08:27:51 -07005465TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005466 const int kWidth = 640;
5467 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005468 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005469
5470 VideoEncoderConfig video_encoder_config;
5471 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5472 // Make format different, to force recreation of encoder.
5473 video_encoder_config.video_format.parameters["foo"] = "foo";
5474 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005475 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005477 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005478
kthelgasonb83797b2017-02-14 11:57:25 -08005479 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005480 video_stream_encoder_->SetSource(&video_source_,
5481 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005482
asaperssonfab67072017-04-04 05:51:49 -07005483 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005484 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005485 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005486
mflodmancc3d4422017-08-03 08:27:51 -07005487 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005488 fake_encoder_.SetQualityScaling(true);
5489}
5490
Åsa Persson139f4dc2019-08-02 09:29:58 +02005491TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5492 webrtc::test::ScopedFieldTrials field_trials(
5493 "WebRTC-Video-QualityScalerSettings/"
5494 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5495 // Reset encoder for field trials to take effect.
5496 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005497 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5498 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005499 const int kWidth = 640;
5500 const int kHeight = 360;
5501
Henrik Boström381d1092020-05-12 18:49:07 +02005502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005503 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005504 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5505 // Frame should not be dropped.
5506 WaitForEncodedFrame(1);
5507
Henrik Boström381d1092020-05-12 18:49:07 +02005508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005509 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5510 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5511 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005512 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5513 // Frame should not be dropped.
5514 WaitForEncodedFrame(2);
5515
Henrik Boström381d1092020-05-12 18:49:07 +02005516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005517 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5518 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5519 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005520 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5521 // Expect to drop this frame, the wait should time out.
5522 ExpectDroppedFrame();
5523
5524 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005525 EXPECT_TRUE_WAIT(
5526 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005527 video_stream_encoder_->Stop();
5528}
5529
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005530TEST_F(VideoStreamEncoderTest,
5531 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5532 webrtc::test::ScopedFieldTrials field_trials(
5533 "WebRTC-Video-QualityScalerSettings/"
5534 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5535 fake_encoder_.SetQualityScaling(false);
5536 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005537 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5538 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005539 const int kWidth = 640;
5540 const int kHeight = 360;
5541
5542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005543 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005544 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5545 // Frame should not be dropped.
5546 WaitForEncodedFrame(1);
5547
5548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5549 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5550 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5551 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5552 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5553 // Frame should not be dropped.
5554 WaitForEncodedFrame(2);
5555
5556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5557 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5558 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5559 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5560 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5561 // Not dropped since quality scaling is disabled.
5562 WaitForEncodedFrame(3);
5563
5564 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005565 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005566 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5567
5568 video_stream_encoder_->Stop();
5569}
5570
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005571TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005572 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005573 // Set simulcast.
5574 ResetEncoder("VP8", 3, 1, 1, false);
5575 fake_encoder_.SetQualityScaling(true);
5576 const int kWidth = 1280;
5577 const int kHeight = 720;
5578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005579 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005580 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5581 // Frame should not be dropped.
5582 WaitForEncodedFrame(1);
5583
5584 // Trigger QVGA "singlecast"
5585 // Update the config.
5586 VideoEncoderConfig video_encoder_config;
5587 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5588 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005589 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005590 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005591 "VP8", /*max qp*/ 56, /*screencast*/ false,
5592 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005593 for (auto& layer : video_encoder_config.simulcast_layers) {
5594 layer.num_temporal_layers = 1;
5595 layer.max_framerate = kDefaultFramerate;
5596 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005597 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005598 video_encoder_config.content_type =
5599 VideoEncoderConfig::ContentType::kRealtimeVideo;
5600
5601 video_encoder_config.simulcast_layers[0].active = true;
5602 video_encoder_config.simulcast_layers[1].active = false;
5603 video_encoder_config.simulcast_layers[2].active = false;
5604
5605 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5606 kMaxPayloadLength);
5607 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5608
5609 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5610 // Frame should not be dropped.
5611 WaitForEncodedFrame(2);
5612
5613 // Trigger HD "singlecast"
5614 video_encoder_config.simulcast_layers[0].active = false;
5615 video_encoder_config.simulcast_layers[1].active = false;
5616 video_encoder_config.simulcast_layers[2].active = true;
5617
5618 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5619 kMaxPayloadLength);
5620 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5621
5622 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5623 // Frame should be dropped because of initial frame drop.
5624 ExpectDroppedFrame();
5625
5626 // Expect the sink_wants to specify a scaled frame.
5627 EXPECT_TRUE_WAIT(
5628 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5629 video_stream_encoder_->Stop();
5630}
5631
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005632TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005633 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005634 // Set simulcast.
5635 ResetEncoder("VP9", 1, 1, 3, false);
5636 fake_encoder_.SetQualityScaling(true);
5637 const int kWidth = 1280;
5638 const int kHeight = 720;
5639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005640 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005641 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5642 // Frame should not be dropped.
5643 WaitForEncodedFrame(1);
5644
5645 // Trigger QVGA "singlecast"
5646 // Update the config.
5647 VideoEncoderConfig video_encoder_config;
5648 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5649 &video_encoder_config);
5650 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5651 vp9_settings.numberOfSpatialLayers = 3;
5652 // Since only one layer is active - automatic resize should be enabled.
5653 vp9_settings.automaticResizeOn = true;
5654 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005655 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005656 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005657 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005658 video_encoder_config.content_type =
5659 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005660 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005661 // which SVC layers are active.
5662 video_encoder_config.simulcast_layers.resize(3);
5663
5664 video_encoder_config.simulcast_layers[0].active = true;
5665 video_encoder_config.simulcast_layers[1].active = false;
5666 video_encoder_config.simulcast_layers[2].active = false;
5667
5668 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5669 kMaxPayloadLength);
5670 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5671
5672 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5673 // Frame should not be dropped.
5674 WaitForEncodedFrame(2);
5675
5676 // Trigger HD "singlecast"
5677 video_encoder_config.simulcast_layers[0].active = false;
5678 video_encoder_config.simulcast_layers[1].active = false;
5679 video_encoder_config.simulcast_layers[2].active = true;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5686 // Frame should be dropped because of initial frame drop.
5687 ExpectDroppedFrame();
5688
5689 // Expect the sink_wants to specify a scaled frame.
5690 EXPECT_TRUE_WAIT(
5691 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5692 video_stream_encoder_->Stop();
5693}
5694
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005695TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005696 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5697 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5698 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5699 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5700 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5701 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5702 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5703 fake_encoder_.SetResolutionBitrateLimits(
5704 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5705
5706 VideoEncoderConfig video_encoder_config;
5707 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5708 &video_encoder_config);
5709 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5710 vp9_settings.numberOfSpatialLayers = 3;
5711 // Since only one layer is active - automatic resize should be enabled.
5712 vp9_settings.automaticResizeOn = true;
5713 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005714 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005715 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005716 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005717 video_encoder_config.content_type =
5718 VideoEncoderConfig::ContentType::kRealtimeVideo;
5719 // Simulcast layers are used to indicate which spatial layers are active.
5720 video_encoder_config.simulcast_layers.resize(3);
5721 video_encoder_config.simulcast_layers[0].active = false;
5722 video_encoder_config.simulcast_layers[1].active = true;
5723 video_encoder_config.simulcast_layers[2].active = false;
5724
5725 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5726 kMaxPayloadLength);
5727 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5728
5729 // The encoder bitrate limits for 360p should be used.
5730 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5731 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005732 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5733 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5734 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5735 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5736 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5737 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005738 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005739 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005740 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005741 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005742
5743 // The encoder bitrate limits for 270p should be used.
5744 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5745 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005746 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5747 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5748 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5749 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5750 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5751 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005752 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005753 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005754 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005755 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005756
5757 video_stream_encoder_->Stop();
5758}
5759
5760TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005761 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5762 VideoEncoderConfig video_encoder_config;
5763 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5764 &video_encoder_config);
5765 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5766 vp9_settings.numberOfSpatialLayers = 3;
5767 // Since only one layer is active - automatic resize should be enabled.
5768 vp9_settings.automaticResizeOn = true;
5769 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005770 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005771 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005772 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005773 video_encoder_config.content_type =
5774 VideoEncoderConfig::ContentType::kRealtimeVideo;
5775 // Simulcast layers are used to indicate which spatial layers are active.
5776 video_encoder_config.simulcast_layers.resize(3);
5777 video_encoder_config.simulcast_layers[0].active = false;
5778 video_encoder_config.simulcast_layers[1].active = true;
5779 video_encoder_config.simulcast_layers[2].active = false;
5780
5781 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5782 kMaxPayloadLength);
5783 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5784
5785 // The default bitrate limits for 360p should be used.
5786 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005787 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5788 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005789 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5790 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005791 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5792 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5793 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5794 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5795 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5796 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005797 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005798 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005799 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005800 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005801
5802 // The default bitrate limits for 270p should be used.
5803 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005804 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5805 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005806 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5807 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005808 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5809 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5810 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5811 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5812 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5813 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005814 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005815 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005816 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005817 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005818
5819 video_stream_encoder_->Stop();
5820}
5821
5822TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5823 webrtc::test::ScopedFieldTrials field_trials(
5824 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5825 VideoEncoderConfig video_encoder_config;
5826 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5827 &video_encoder_config);
5828 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5829 vp9_settings.numberOfSpatialLayers = 3;
5830 // Since only one layer is active - automatic resize should be enabled.
5831 vp9_settings.automaticResizeOn = true;
5832 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005833 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005834 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005835 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005836 video_encoder_config.content_type =
5837 VideoEncoderConfig::ContentType::kRealtimeVideo;
5838 // Simulcast layers are used to indicate which spatial layers are active.
5839 video_encoder_config.simulcast_layers.resize(3);
5840 video_encoder_config.simulcast_layers[0].active = false;
5841 video_encoder_config.simulcast_layers[1].active = true;
5842 video_encoder_config.simulcast_layers[2].active = false;
5843
5844 // Reset encoder for field trials to take effect.
5845 ConfigureEncoder(video_encoder_config.Copy());
5846
5847 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5848 kMaxPayloadLength);
5849 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5850
5851 // The default bitrate limits for 360p should not be used.
5852 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005853 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5854 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005855 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5856 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005857 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5858 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5859 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5860 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5861 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5862 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005863 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005864 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005865
5866 video_stream_encoder_->Stop();
5867}
5868
5869TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5870 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5871 /*num_spatial_layers=*/1, /*screenshare=*/false);
5872
5873 // The default singlecast bitrate limits for 720p should not be used.
5874 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005875 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5876 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005877 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5878 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005879 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5880 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5881 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5882 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5883 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5884 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005885 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005886 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005887
5888 video_stream_encoder_->Stop();
5889}
5890
5891TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005892 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5893 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5894 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5895 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5896 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5897 fake_encoder_.SetResolutionBitrateLimits(
5898 {kEncoderLimits180p, kEncoderLimits720p});
5899
5900 VideoEncoderConfig video_encoder_config;
5901 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5902 &video_encoder_config);
5903 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5904 vp9_settings.numberOfSpatialLayers = 3;
5905 // Since only one layer is active - automatic resize should be enabled.
5906 vp9_settings.automaticResizeOn = true;
5907 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005908 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005909 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005910 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005911 video_encoder_config.content_type =
5912 VideoEncoderConfig::ContentType::kRealtimeVideo;
5913 // Simulcast layers are used to indicate which spatial layers are active.
5914 video_encoder_config.simulcast_layers.resize(3);
5915 video_encoder_config.simulcast_layers[0].active = true;
5916 video_encoder_config.simulcast_layers[1].active = false;
5917 video_encoder_config.simulcast_layers[2].active = false;
5918
5919 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5920 kMaxPayloadLength);
5921 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5922
5923 // Limits not applied on lowest stream, limits for 180p should not be used.
5924 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5925 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005926 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5927 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5928 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5929 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5930 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5931 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005932 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005933 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005934 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005935 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005936
5937 video_stream_encoder_->Stop();
5938}
5939
5940TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005941 InitialFrameDropActivatesWhenResolutionIncreases) {
5942 const int kWidth = 640;
5943 const int kHeight = 360;
5944
5945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005946 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005947 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5948 // Frame should not be dropped.
5949 WaitForEncodedFrame(1);
5950
5951 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005952 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005953 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5954 // Frame should not be dropped, bitrate not too low for frame.
5955 WaitForEncodedFrame(2);
5956
5957 // Incoming resolution increases.
5958 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5959 // Expect to drop this frame, bitrate too low for frame.
5960 ExpectDroppedFrame();
5961
5962 // Expect the sink_wants to specify a scaled frame.
5963 EXPECT_TRUE_WAIT(
5964 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5965 video_stream_encoder_->Stop();
5966}
5967
5968TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5969 const int kWidth = 640;
5970 const int kHeight = 360;
5971 // So that quality scaling doesn't happen by itself.
5972 fake_encoder_.SetQp(kQpHigh);
5973
5974 AdaptingFrameForwarder source(&time_controller_);
5975 source.set_adaptation_enabled(true);
5976 video_stream_encoder_->SetSource(
5977 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5978
5979 int timestamp = 1;
5980
5981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005982 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005983 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5984 WaitForEncodedFrame(timestamp);
5985 timestamp += 9000;
5986 // Long pause to disable all first BWE drop logic.
5987 AdvanceTime(TimeDelta::Millis(1000));
5988
5989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005990 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005991 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5992 // Not dropped frame, as initial frame drop is disabled by now.
5993 WaitForEncodedFrame(timestamp);
5994 timestamp += 9000;
5995 AdvanceTime(TimeDelta::Millis(100));
5996
5997 // Quality adaptation down.
5998 video_stream_encoder_->TriggerQualityLow();
5999
6000 // Adaptation has an effect.
6001 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6002 5000);
6003
6004 // Frame isn't dropped as initial frame dropper is disabled.
6005 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6006 WaitForEncodedFrame(timestamp);
6007 timestamp += 9000;
6008 AdvanceTime(TimeDelta::Millis(100));
6009
6010 // Quality adaptation up.
6011 video_stream_encoder_->TriggerQualityHigh();
6012
6013 // Adaptation has an effect.
6014 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6015 5000);
6016
6017 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6018 // Frame should not be dropped, as initial framedropper is off.
6019 WaitForEncodedFrame(timestamp);
6020
6021 video_stream_encoder_->Stop();
6022}
6023
Åsa Persson7f354f82021-02-04 15:52:15 +01006024TEST_F(VideoStreamEncoderTest,
6025 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6026 const int kMinStartBps360p = 222000;
6027 fake_encoder_.SetResolutionBitrateLimits(
6028 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6029 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6030 800000)});
6031
6032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6033 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6034 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6035 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6036 0, 0, 0);
6037 // Frame should not be dropped, bitrate not too low for frame.
6038 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6039 WaitForEncodedFrame(1);
6040
6041 // Incoming resolution increases, initial frame drop activates.
6042 // Frame should be dropped, link allocation too low for frame.
6043 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6044 ExpectDroppedFrame();
6045
6046 // Expect sink_wants to specify a scaled frame.
6047 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6048 5000);
6049 video_stream_encoder_->Stop();
6050}
6051
6052TEST_F(VideoStreamEncoderTest,
6053 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6054 const int kMinStartBps360p = 222000;
6055 fake_encoder_.SetResolutionBitrateLimits(
6056 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6057 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6058 800000)});
6059
6060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6061 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6062 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6063 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6064 0, 0, 0);
6065 // Frame should not be dropped, bitrate not too low for frame.
6066 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6067 WaitForEncodedFrame(1);
6068
6069 // Incoming resolution increases, initial frame drop activates.
6070 // Frame should be dropped, link allocation not too low for frame.
6071 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6072 WaitForEncodedFrame(2);
6073
6074 video_stream_encoder_->Stop();
6075}
6076
Åsa Perssone644a032019-11-08 15:56:00 +01006077TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6078 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006079 "WebRTC-Video-QualityRampupSettings/"
6080 "min_pixels:921600,min_duration_ms:2000/");
6081
6082 const int kWidth = 1280;
6083 const int kHeight = 720;
6084 const int kFps = 10;
6085 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006086
6087 // Reset encoder for field trials to take effect.
6088 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006089 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006090 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006091 ConfigureEncoder(std::move(config));
6092 fake_encoder_.SetQp(kQpLow);
6093
6094 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006095 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006096 source.set_adaptation_enabled(true);
6097 video_stream_encoder_->SetSource(&source,
6098 DegradationPreference::MAINTAIN_FRAMERATE);
6099
6100 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006101 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006102 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006103 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006104
6105 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006106 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006107 int64_t timestamp_ms = kFrameIntervalMs;
6108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6109 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006110 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6111 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006112
6113 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6115 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006116
Artem Titovab30d722021-07-27 16:22:11 +02006117 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006118 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006119 for (size_t i = 1; i <= 10; i++) {
6120 timestamp_ms += kFrameIntervalMs;
6121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6122 WaitForEncodedFrame(timestamp_ms);
6123 }
Åsa Persson06defc42021-09-10 15:28:48 +02006124
6125 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6126 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6127 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6128 timestamp_ms += kFrameIntervalMs;
6129 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6130 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006131 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6132 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6133
Åsa Persson06defc42021-09-10 15:28:48 +02006134 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006135 timestamp_ms += kFrameIntervalMs;
6136 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6137 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006138 // The ramp-up code involves the adaptation queue, give it time to execute.
6139 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006140 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006141 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006142
6143 // Frame should not be adapted.
6144 timestamp_ms += kFrameIntervalMs;
6145 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6146 WaitForEncodedFrame(kWidth, kHeight);
6147 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6148
6149 video_stream_encoder_->Stop();
6150}
6151
mflodmancc3d4422017-08-03 08:27:51 -07006152TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006153 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006154 webrtc::test::ScopedFieldTrials field_trials(
6155 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006156 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006157 source.set_adaptation_enabled(true);
6158 video_stream_encoder_->SetSource(&source,
6159 DegradationPreference::MAINTAIN_FRAMERATE);
6160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006161 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006162 fake_encoder_.SetQp(kQpHigh + 1);
6163 const int kWidth = 1280;
6164 const int kHeight = 720;
6165 const int64_t kFrameIntervalMs = 100;
6166 int64_t timestamp_ms = kFrameIntervalMs;
6167 for (size_t i = 1; i <= 100; i++) {
6168 timestamp_ms += kFrameIntervalMs;
6169 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6170 WaitForEncodedFrame(timestamp_ms);
6171 }
6172 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6173 // for the first time.
6174 // TODO(eshr): We should avoid these waits by using threads with simulated
6175 // time.
6176 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6177 2000 * 2.5 * 2);
6178 timestamp_ms += kFrameIntervalMs;
6179 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6180 WaitForEncodedFrame(timestamp_ms);
6181 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6182 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6183 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6184
6185 // Disable Quality scaling by turning off scaler on the encoder and
6186 // reconfiguring.
6187 fake_encoder_.SetQualityScaling(false);
6188 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6189 kMaxPayloadLength);
6190 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006191 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006192 // Since we turned off the quality scaler, the adaptations made by it are
6193 // removed.
6194 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6195 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6196
6197 video_stream_encoder_->Stop();
6198}
6199
6200TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006201 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6202 const int kTooSmallWidth = 10;
6203 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006205 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006206
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006207 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006208 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006209 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006210 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006211 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006212 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6213
6214 // Trigger adapt down, too small frame, expect no change.
6215 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006216 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006217 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006218 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006219 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6220 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6221
mflodmancc3d4422017-08-03 08:27:51 -07006222 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006223}
6224
mflodmancc3d4422017-08-03 08:27:51 -07006225TEST_F(VideoStreamEncoderTest,
6226 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006227 const int kTooSmallWidth = 10;
6228 const int kTooSmallHeight = 10;
6229 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006230 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006231 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006232
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006233 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006234 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006235 video_stream_encoder_->SetSource(&source,
6236 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006237 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006238 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6239 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6240
6241 // Trigger adapt down, expect limited framerate.
6242 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006243 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006244 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006245 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6247 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6248 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6249
6250 // Trigger adapt down, too small frame, expect no change.
6251 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006252 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006253 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006254 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006255 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6256 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6257 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6258
mflodmancc3d4422017-08-03 08:27:51 -07006259 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006260}
6261
mflodmancc3d4422017-08-03 08:27:51 -07006262TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006263 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006265 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006266 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006267 const int kFrameWidth = 1280;
6268 const int kFrameHeight = 720;
6269 video_source_.IncomingCapturedFrame(
6270 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006271 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006272 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006273}
6274
sprangb1ca0732017-02-01 08:38:12 -08006275// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006276TEST_F(VideoStreamEncoderTest,
6277 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006279 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006280
6281 const int kFrameWidth = 1280;
6282 const int kFrameHeight = 720;
6283 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006284 // requested by
6285 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006286 video_source_.set_adaptation_enabled(true);
6287
6288 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006289 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006290 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006291
6292 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006293 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006294 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006295 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006296 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006297
asaperssonfab67072017-04-04 05:51:49 -07006298 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006299 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006300 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006301 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006302 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006303
mflodmancc3d4422017-08-03 08:27:51 -07006304 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006305}
sprangfe627f32017-03-29 08:24:59 -07006306
mflodmancc3d4422017-08-03 08:27:51 -07006307TEST_F(VideoStreamEncoderTest,
6308 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006309 const int kFrameWidth = 1280;
6310 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006311
Henrik Boström381d1092020-05-12 18:49:07 +02006312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006313 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006314 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006315 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006316 video_source_.set_adaptation_enabled(true);
6317
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006318 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006319
6320 video_source_.IncomingCapturedFrame(
6321 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006322 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006323
6324 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006325 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006326
6327 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006328 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006329 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006330 video_source_.IncomingCapturedFrame(
6331 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006332 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006333 }
6334
6335 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006336 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006337 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006338 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006339 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006340 video_source_.IncomingCapturedFrame(
6341 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006342 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006343 ++num_frames_dropped;
6344 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006345 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006346 }
6347 }
6348
sprang4847ae62017-06-27 07:06:52 -07006349 // Add some slack to account for frames dropped by the frame dropper.
6350 const int kErrorMargin = 1;
6351 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006352 kErrorMargin);
6353
6354 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006355 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006356 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006357 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006358 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006359 video_source_.IncomingCapturedFrame(
6360 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006361 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006362 ++num_frames_dropped;
6363 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006364 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006365 }
6366 }
sprang4847ae62017-06-27 07:06:52 -07006367 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006368 kErrorMargin);
6369
6370 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006371 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006372 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006373 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006374 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006375 video_source_.IncomingCapturedFrame(
6376 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006377 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006378 ++num_frames_dropped;
6379 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006380 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006381 }
6382 }
sprang4847ae62017-06-27 07:06:52 -07006383 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006384 kErrorMargin);
6385
6386 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006387 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006388 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006389 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006390 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006391 video_source_.IncomingCapturedFrame(
6392 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006393 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006394 ++num_frames_dropped;
6395 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006396 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006397 }
6398 }
6399 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6400
mflodmancc3d4422017-08-03 08:27:51 -07006401 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006402}
6403
mflodmancc3d4422017-08-03 08:27:51 -07006404TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006405 const int kFramerateFps = 5;
6406 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006407 const int kFrameWidth = 1280;
6408 const int kFrameHeight = 720;
6409
sprang4847ae62017-06-27 07:06:52 -07006410 // Reconfigure encoder with two temporal layers and screensharing, which will
6411 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006412 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006413
Henrik Boström381d1092020-05-12 18:49:07 +02006414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006415 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006416 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006417 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006418 video_source_.set_adaptation_enabled(true);
6419
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006420 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006421
6422 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006423 rtc::VideoSinkWants last_wants;
6424 do {
6425 last_wants = video_source_.sink_wants();
6426
sprangc5d62e22017-04-02 23:53:04 -07006427 // Insert frames to get a new fps estimate...
6428 for (int j = 0; j < kFramerateFps; ++j) {
6429 video_source_.IncomingCapturedFrame(
6430 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006431 if (video_source_.last_sent_width()) {
6432 sink_.WaitForEncodedFrame(timestamp_ms);
6433 }
sprangc5d62e22017-04-02 23:53:04 -07006434 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006435 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006436 }
6437 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006438 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006439 } while (video_source_.sink_wants().max_framerate_fps <
6440 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006441
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006442 EXPECT_THAT(video_source_.sink_wants(),
6443 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006444
mflodmancc3d4422017-08-03 08:27:51 -07006445 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006446}
asaperssonf7e294d2017-06-13 23:25:22 -07006447
mflodmancc3d4422017-08-03 08:27:51 -07006448TEST_F(VideoStreamEncoderTest,
6449 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006450 const int kWidth = 1280;
6451 const int kHeight = 720;
6452 const int64_t kFrameIntervalMs = 150;
6453 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006455 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006456
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006457 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006458 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006459 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006460 video_stream_encoder_->SetSource(&source,
6461 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006462 timestamp_ms += kFrameIntervalMs;
6463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006464 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006465 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6468 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6469
6470 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006471 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006472 timestamp_ms += kFrameIntervalMs;
6473 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006474 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006475 EXPECT_THAT(source.sink_wants(),
6476 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006477 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6479 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6480
6481 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006482 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006483 timestamp_ms += kFrameIntervalMs;
6484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006486 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6488 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6489 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6490
6491 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006492 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006493 timestamp_ms += kFrameIntervalMs;
6494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006495 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006496 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006497 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6499 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6500
6501 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006502 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006503 timestamp_ms += kFrameIntervalMs;
6504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006505 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006506 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006507 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6509 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6510
6511 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006512 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006513 timestamp_ms += kFrameIntervalMs;
6514 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006515 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006516 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6519 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6520
6521 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006522 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006523 timestamp_ms += kFrameIntervalMs;
6524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006525 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006526 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6529 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6530
6531 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006532 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006533 timestamp_ms += kFrameIntervalMs;
6534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006535 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006536 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006537 rtc::VideoSinkWants last_wants = source.sink_wants();
6538 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6540 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6541
6542 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006543 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006544 timestamp_ms += kFrameIntervalMs;
6545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006546 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006547 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006548 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6550 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6551
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006552 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006553 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006554 timestamp_ms += kFrameIntervalMs;
6555 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006556 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006557 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006558 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6560 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6561
6562 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006563 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006564 timestamp_ms += kFrameIntervalMs;
6565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006566 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006567 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6570 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6571
6572 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006573 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006574 timestamp_ms += kFrameIntervalMs;
6575 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006576 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006577 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6580 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6581
6582 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006583 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006584 timestamp_ms += kFrameIntervalMs;
6585 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006586 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006587 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6589 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6590 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6591
6592 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006593 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006594 timestamp_ms += kFrameIntervalMs;
6595 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006596 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006597 EXPECT_THAT(source.sink_wants(), FpsMax());
6598 EXPECT_EQ(source.sink_wants().max_pixel_count,
6599 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6602 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6603
6604 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006605 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006606 timestamp_ms += kFrameIntervalMs;
6607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006608 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006609 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6611 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6612 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6613
Åsa Persson30ab0152019-08-27 12:22:33 +02006614 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006615 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006616 timestamp_ms += kFrameIntervalMs;
6617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006618 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006619 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006620 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006621 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6622 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6623 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6624
6625 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006626 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006627 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006628 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6629
mflodmancc3d4422017-08-03 08:27:51 -07006630 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006631}
6632
mflodmancc3d4422017-08-03 08:27:51 -07006633TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006634 const int kWidth = 1280;
6635 const int kHeight = 720;
6636 const int64_t kFrameIntervalMs = 150;
6637 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006638 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006639 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006640
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006641 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006642 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006643 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006644 video_stream_encoder_->SetSource(&source,
6645 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006646 timestamp_ms += kFrameIntervalMs;
6647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006648 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006649 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6653 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6654 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6655 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6656
6657 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006658 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006659 timestamp_ms += kFrameIntervalMs;
6660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006661 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006662 EXPECT_THAT(source.sink_wants(),
6663 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6665 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6666 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6667 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6668 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6669 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6670
6671 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006672 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006673 timestamp_ms += kFrameIntervalMs;
6674 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006675 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006676 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006677 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6679 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6681 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6682 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6683
6684 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006685 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006686 timestamp_ms += kFrameIntervalMs;
6687 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006688 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006689 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006690 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006691 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6692 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6693 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6694 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6695 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6696
Evan Shrubsole64469032020-06-11 10:45:29 +02006697 // Trigger cpu adapt up, expect no change since QP is most limited.
6698 {
6699 // Store current sink wants since we expect no change and if there is no
6700 // change then last_wants() is not updated.
6701 auto previous_sink_wants = source.sink_wants();
6702 video_stream_encoder_->TriggerCpuUnderuse();
6703 timestamp_ms += kFrameIntervalMs;
6704 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6705 WaitForEncodedFrame(timestamp_ms);
6706 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6707 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6708 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6709 }
6710
6711 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6712 video_stream_encoder_->TriggerQualityHigh();
6713 timestamp_ms += kFrameIntervalMs;
6714 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6715 WaitForEncodedFrame(timestamp_ms);
6716 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6717 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6718 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6719 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6721 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6722 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6723
6724 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6725 // expect increased resolution (960x540@30fps).
6726 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006727 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006728 timestamp_ms += kFrameIntervalMs;
6729 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006730 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006731 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006732 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6733 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6734 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6735 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6736 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006737 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006738
Evan Shrubsole64469032020-06-11 10:45:29 +02006739 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6740 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006741 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006742 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006743 timestamp_ms += kFrameIntervalMs;
6744 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006745 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006746 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006747 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006748 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6750 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6752 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006753 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006754
6755 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006756 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006758 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006759 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006760
mflodmancc3d4422017-08-03 08:27:51 -07006761 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006762}
6763
mflodmancc3d4422017-08-03 08:27:51 -07006764TEST_F(VideoStreamEncoderTest,
6765 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006766 const int kWidth = 640;
6767 const int kHeight = 360;
6768 const int kFpsLimit = 15;
6769 const int64_t kFrameIntervalMs = 150;
6770 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006773
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006774 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006775 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006776 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006777 video_stream_encoder_->SetSource(&source,
6778 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006779 timestamp_ms += kFrameIntervalMs;
6780 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006781 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006782 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6784 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6786 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6787 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6788 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6789
6790 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006791 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006792 timestamp_ms += kFrameIntervalMs;
6793 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006794 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006795 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6799 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6800 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6801 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6802
6803 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006804 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006805 timestamp_ms += kFrameIntervalMs;
6806 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006807 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006808 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006809 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006810 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6812 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6813 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6814 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6815
Evan Shrubsole64469032020-06-11 10:45:29 +02006816 // Trigger cpu adapt up, expect no change because quality is most limited.
6817 {
6818 auto previous_sink_wants = source.sink_wants();
6819 // Store current sink wants since we expect no change ind if there is no
6820 // change then last__wants() is not updated.
6821 video_stream_encoder_->TriggerCpuUnderuse();
6822 timestamp_ms += kFrameIntervalMs;
6823 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6824 WaitForEncodedFrame(timestamp_ms);
6825 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6826 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6827 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6828 }
6829
6830 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6831 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006832 timestamp_ms += kFrameIntervalMs;
6833 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006834 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006835 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006836 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6837 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6838 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006839 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6840 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6841 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006842
Evan Shrubsole64469032020-06-11 10:45:29 +02006843 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006844 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006845 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006846 timestamp_ms += kFrameIntervalMs;
6847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006848 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006849 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006850 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6852 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6854 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006855 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006856
6857 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006858 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006859 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006860 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006861 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006862
mflodmancc3d4422017-08-03 08:27:51 -07006863 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006864}
6865
mflodmancc3d4422017-08-03 08:27:51 -07006866TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006867 const int kFrameWidth = 1920;
6868 const int kFrameHeight = 1080;
6869 // 3/4 of 1920.
6870 const int kAdaptedFrameWidth = 1440;
6871 // 3/4 of 1080 rounded down to multiple of 4.
6872 const int kAdaptedFrameHeight = 808;
6873 const int kFramerate = 24;
6874
Henrik Boström381d1092020-05-12 18:49:07 +02006875 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006876 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006877 // Trigger reconfigure encoder (without resetting the entire instance).
6878 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006879 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6880 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006881 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006882 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006883 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006884 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006885 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006886 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006887
6888 video_source_.set_adaptation_enabled(true);
6889
6890 video_source_.IncomingCapturedFrame(
6891 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006892 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006893
6894 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006895 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006896 video_source_.IncomingCapturedFrame(
6897 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006898 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006899
mflodmancc3d4422017-08-03 08:27:51 -07006900 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006901}
6902
mflodmancc3d4422017-08-03 08:27:51 -07006903TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006904 const int kFrameWidth = 1280;
6905 const int kFrameHeight = 720;
6906 const int kLowFps = 2;
6907 const int kHighFps = 30;
6908
Henrik Boström381d1092020-05-12 18:49:07 +02006909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006910 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006911
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006912 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006913 max_framerate_ = kLowFps;
6914
6915 // Insert 2 seconds of 2fps video.
6916 for (int i = 0; i < kLowFps * 2; ++i) {
6917 video_source_.IncomingCapturedFrame(
6918 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6919 WaitForEncodedFrame(timestamp_ms);
6920 timestamp_ms += 1000 / kLowFps;
6921 }
6922
6923 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006925 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006926 video_source_.IncomingCapturedFrame(
6927 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6928 WaitForEncodedFrame(timestamp_ms);
6929 timestamp_ms += 1000 / kLowFps;
6930
6931 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6932
6933 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006934 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006935 const int kFrameIntervalMs = 1000 / kHighFps;
6936 max_framerate_ = kHighFps;
6937 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6938 video_source_.IncomingCapturedFrame(
6939 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6940 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6941 // be dropped if the encoder hans't been updated with the new higher target
6942 // framerate yet, causing it to overshoot the target bitrate and then
6943 // suffering the wrath of the media optimizer.
6944 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6945 timestamp_ms += kFrameIntervalMs;
6946 }
6947
6948 // Don expect correct measurement just yet, but it should be higher than
6949 // before.
6950 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6951
mflodmancc3d4422017-08-03 08:27:51 -07006952 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006953}
6954
mflodmancc3d4422017-08-03 08:27:51 -07006955TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006956 const int kFrameWidth = 1280;
6957 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006958 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006959 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006960 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006961
Henrik Boström381d1092020-05-12 18:49:07 +02006962 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006963 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006964 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006965
6966 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006967 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006968 video_source_.IncomingCapturedFrame(
6969 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6970 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006971 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006972
6973 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006975 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006976
6977 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006978 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006979 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006980
Per Kjellanderdcef6412020-10-07 15:09:05 +02006981 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006982 video_source_.IncomingCapturedFrame(
6983 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6984 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006985 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006986
mflodmancc3d4422017-08-03 08:27:51 -07006987 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006988}
ilnik6b826ef2017-06-16 06:53:48 -07006989
Niels Möller4db138e2018-04-19 09:04:13 +02006990TEST_F(VideoStreamEncoderTest,
6991 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6992 const int kFrameWidth = 1280;
6993 const int kFrameHeight = 720;
6994 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006996 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006997 video_source_.IncomingCapturedFrame(
6998 CreateFrame(1, kFrameWidth, kFrameHeight));
6999 WaitForEncodedFrame(1);
7000 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7001 .low_encode_usage_threshold_percent,
7002 default_options.low_encode_usage_threshold_percent);
7003 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7004 .high_encode_usage_threshold_percent,
7005 default_options.high_encode_usage_threshold_percent);
7006 video_stream_encoder_->Stop();
7007}
7008
7009TEST_F(VideoStreamEncoderTest,
7010 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7011 const int kFrameWidth = 1280;
7012 const int kFrameHeight = 720;
7013 CpuOveruseOptions hardware_options;
7014 hardware_options.low_encode_usage_threshold_percent = 150;
7015 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007016 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007017
Henrik Boström381d1092020-05-12 18:49:07 +02007018 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007019 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007020 video_source_.IncomingCapturedFrame(
7021 CreateFrame(1, kFrameWidth, kFrameHeight));
7022 WaitForEncodedFrame(1);
7023 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7024 .low_encode_usage_threshold_percent,
7025 hardware_options.low_encode_usage_threshold_percent);
7026 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7027 .high_encode_usage_threshold_percent,
7028 hardware_options.high_encode_usage_threshold_percent);
7029 video_stream_encoder_->Stop();
7030}
7031
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007032TEST_F(VideoStreamEncoderTest,
7033 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7034 const int kFrameWidth = 1280;
7035 const int kFrameHeight = 720;
7036
7037 const CpuOveruseOptions default_options;
7038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007040 video_source_.IncomingCapturedFrame(
7041 CreateFrame(1, kFrameWidth, kFrameHeight));
7042 WaitForEncodedFrame(1);
7043 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7044 .low_encode_usage_threshold_percent,
7045 default_options.low_encode_usage_threshold_percent);
7046 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7047 .high_encode_usage_threshold_percent,
7048 default_options.high_encode_usage_threshold_percent);
7049
7050 CpuOveruseOptions hardware_options;
7051 hardware_options.low_encode_usage_threshold_percent = 150;
7052 hardware_options.high_encode_usage_threshold_percent = 200;
7053 fake_encoder_.SetIsHardwareAccelerated(true);
7054
7055 video_source_.IncomingCapturedFrame(
7056 CreateFrame(2, kFrameWidth, kFrameHeight));
7057 WaitForEncodedFrame(2);
7058
7059 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7060 .low_encode_usage_threshold_percent,
7061 hardware_options.low_encode_usage_threshold_percent);
7062 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7063 .high_encode_usage_threshold_percent,
7064 hardware_options.high_encode_usage_threshold_percent);
7065
7066 video_stream_encoder_->Stop();
7067}
7068
Niels Möller6bb5ab92019-01-11 11:11:10 +01007069TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7070 const int kFrameWidth = 320;
7071 const int kFrameHeight = 240;
7072 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007073 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007074 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7075
Henrik Boström381d1092020-05-12 18:49:07 +02007076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007077 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007078
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007079 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007080 max_framerate_ = kFps;
7081
7082 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7083 fake_encoder_.SimulateOvershoot(1.0);
7084 int num_dropped = 0;
7085 for (int i = 0; i < kNumFramesInRun; ++i) {
7086 video_source_.IncomingCapturedFrame(
7087 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7088 // Wait up to two frame durations for a frame to arrive.
7089 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7090 ++num_dropped;
7091 }
7092 timestamp_ms += 1000 / kFps;
7093 }
7094
Erik Språnga8d48ab2019-02-08 14:17:40 +01007095 // Framerate should be measured to be near the expected target rate.
7096 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7097
7098 // Frame drops should be within 5% of expected 0%.
7099 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007100
7101 // Make encoder produce frames at double the expected bitrate during 3 seconds
7102 // of video, verify number of drops. Rate needs to be slightly changed in
7103 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007104 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007105 const RateControlSettings trials =
7106 RateControlSettings::ParseFromFieldTrials();
7107 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007108 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007109 // frame dropping since the adjuter will try to just lower the target
7110 // bitrate rather than drop frames. If network headroom can be used, it
7111 // doesn't push back as hard so we don't need quite as much overshoot.
7112 // These numbers are unfortunately a bit magical but there's not trivial
7113 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007114 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007115 }
7116 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007118 kTargetBitrate + DataRate::KilobitsPerSec(1),
7119 kTargetBitrate + DataRate::KilobitsPerSec(1),
7120 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007121 num_dropped = 0;
7122 for (int i = 0; i < kNumFramesInRun; ++i) {
7123 video_source_.IncomingCapturedFrame(
7124 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7125 // Wait up to two frame durations for a frame to arrive.
7126 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7127 ++num_dropped;
7128 }
7129 timestamp_ms += 1000 / kFps;
7130 }
7131
Henrik Boström381d1092020-05-12 18:49:07 +02007132 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007133 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007134
7135 // Target framerate should be still be near the expected target, despite
7136 // the frame drops.
7137 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7138
7139 // Frame drops should be within 5% of expected 50%.
7140 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007141
7142 video_stream_encoder_->Stop();
7143}
7144
7145TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7146 const int kFrameWidth = 320;
7147 const int kFrameHeight = 240;
7148 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007149 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007150
7151 ASSERT_GT(max_framerate_, kActualInputFps);
7152
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007153 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007154 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007155 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007156 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007157
7158 // Insert 3 seconds of video, with an input fps lower than configured max.
7159 for (int i = 0; i < kActualInputFps * 3; ++i) {
7160 video_source_.IncomingCapturedFrame(
7161 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7162 // Wait up to two frame durations for a frame to arrive.
7163 WaitForEncodedFrame(timestamp_ms);
7164 timestamp_ms += 1000 / kActualInputFps;
7165 }
7166
7167 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7168
7169 video_stream_encoder_->Stop();
7170}
7171
Markus Handell9a478b52021-11-18 16:07:01 +01007172TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007173 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007174 test::FrameForwarder source;
7175 video_stream_encoder_->SetSource(&source,
7176 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007177 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007178 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007179
Markus Handell9a478b52021-11-18 16:07:01 +01007180 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007181 WaitForEncodedFrame(1);
7182 // On the very first frame full update should be forced.
7183 rect = fake_encoder_.GetLastUpdateRect();
7184 EXPECT_EQ(rect.offset_x, 0);
7185 EXPECT_EQ(rect.offset_y, 0);
7186 EXPECT_EQ(rect.height, codec_height_);
7187 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007188 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7189 // scheduled for processing during encoder queue processing of frame 2.
7190 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7191 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007192 WaitForEncodedFrame(3);
7193 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7194 rect = fake_encoder_.GetLastUpdateRect();
7195 EXPECT_EQ(rect.offset_x, 1);
7196 EXPECT_EQ(rect.offset_y, 0);
7197 EXPECT_EQ(rect.width, 10);
7198 EXPECT_EQ(rect.height, 1);
7199
Markus Handell9a478b52021-11-18 16:07:01 +01007200 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007201 WaitForEncodedFrame(4);
7202 // Previous frame was encoded, so no accumulation should happen.
7203 rect = fake_encoder_.GetLastUpdateRect();
7204 EXPECT_EQ(rect.offset_x, 0);
7205 EXPECT_EQ(rect.offset_y, 0);
7206 EXPECT_EQ(rect.width, 1);
7207 EXPECT_EQ(rect.height, 1);
7208
7209 video_stream_encoder_->Stop();
7210}
7211
Erik Språngd7329ca2019-02-21 21:19:53 +01007212TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007213 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007214 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007215
7216 // First frame is always keyframe.
7217 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7218 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007219 EXPECT_THAT(
7220 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007221 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007222
7223 // Insert delta frame.
7224 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7225 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007226 EXPECT_THAT(
7227 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007228 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007229
7230 // Request next frame be a key-frame.
7231 video_stream_encoder_->SendKeyFrame();
7232 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7233 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007234 EXPECT_THAT(
7235 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007236 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007237
7238 video_stream_encoder_->Stop();
7239}
7240
7241TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7242 // Setup simulcast with three streams.
7243 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007245 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7246 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007247 // Wait for all three layers before triggering event.
7248 sink_.SetNumExpectedLayers(3);
7249
7250 // First frame is always keyframe.
7251 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7252 WaitForEncodedFrame(1);
7253 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007254 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7255 VideoFrameType::kVideoFrameKey,
7256 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007257
7258 // Insert delta frame.
7259 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7260 WaitForEncodedFrame(2);
7261 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007262 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7263 VideoFrameType::kVideoFrameDelta,
7264 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007265
7266 // Request next frame be a key-frame.
7267 // Only first stream is configured to produce key-frame.
7268 video_stream_encoder_->SendKeyFrame();
7269 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7270 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007271
7272 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7273 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007274 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007275 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007276 VideoFrameType::kVideoFrameKey,
7277 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007278
7279 video_stream_encoder_->Stop();
7280}
7281
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007282TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007283 // SPS contains VUI with restrictions on the maximum number of reordered
7284 // pictures, there is no need to rewrite the bitstream to enable faster
7285 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007286 ResetEncoder("H264", 1, 1, 1, false);
7287
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007289 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007291
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007292 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007293 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007294
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007295 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7296 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007297
7298 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007299 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007300
7301 video_stream_encoder_->Stop();
7302}
7303
7304TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007305 // SPS does not contain VUI, the bitstream is will be rewritten with added
7306 // VUI with restrictions on the maximum number of reordered pictures to
7307 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007308 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7309 0x00, 0x00, 0x03, 0x03, 0xF4,
7310 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007311 ResetEncoder("H264", 1, 1, 1, false);
7312
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007314 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007315 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007316
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007317 fake_encoder_.SetEncodedImageData(
7318 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007319
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007320 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7321 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007322
7323 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007324 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007325
7326 video_stream_encoder_->Stop();
7327}
7328
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007329TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7330 const int kFrameWidth = 1280;
7331 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007332 const DataRate kTargetBitrate =
7333 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007334
Henrik Boström381d1092020-05-12 18:49:07 +02007335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007336 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007337 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7338
7339 // Insert a first video frame. It should be dropped because of downscale in
7340 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007341 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007342 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7343 frame.set_rotation(kVideoRotation_270);
7344 video_source_.IncomingCapturedFrame(frame);
7345
7346 ExpectDroppedFrame();
7347
7348 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007349 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007350 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7351 frame.set_rotation(kVideoRotation_90);
7352 video_source_.IncomingCapturedFrame(frame);
7353
7354 WaitForEncodedFrame(timestamp_ms);
7355 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7356
7357 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007358 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007359 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7360 frame.set_rotation(kVideoRotation_180);
7361 video_source_.IncomingCapturedFrame(frame);
7362
7363 WaitForEncodedFrame(timestamp_ms);
7364 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7365
7366 video_stream_encoder_->Stop();
7367}
7368
Erik Språng5056af02019-09-02 15:53:11 +02007369TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7370 const int kFrameWidth = 320;
7371 const int kFrameHeight = 180;
7372
7373 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007374 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007375 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7376 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7377 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007378 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007379 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007380 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007381
7382 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007383 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007384 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7385 frame.set_rotation(kVideoRotation_270);
7386 video_source_.IncomingCapturedFrame(frame);
7387 WaitForEncodedFrame(timestamp_ms);
7388
7389 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007390 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007391 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7392 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007393 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007394 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007395 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007396 /*link_allocation=*/target_rate,
7397 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007398 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007399 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007400 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7401
7402 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7403 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7404 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007405 DataRate allocation_sum =
7406 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007407 EXPECT_EQ(min_rate, allocation_sum);
7408 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7409
7410 video_stream_encoder_->Stop();
7411}
7412
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007413TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007415 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007416 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007417 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007418 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7419 WaitForEncodedFrame(1);
7420
7421 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7422 ASSERT_TRUE(prev_rate_settings.has_value());
7423 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7424 kDefaultFramerate);
7425
7426 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7427 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7428 timestamp_ms += 1000 / kDefaultFramerate;
7429 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7430 WaitForEncodedFrame(timestamp_ms);
7431 }
7432 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7433 kDefaultFramerate);
7434 // Capture larger frame to trigger a reconfigure.
7435 codec_height_ *= 2;
7436 codec_width_ *= 2;
7437 timestamp_ms += 1000 / kDefaultFramerate;
7438 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7439 WaitForEncodedFrame(timestamp_ms);
7440
7441 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7442 auto current_rate_settings =
7443 fake_encoder_.GetAndResetLastRateControlSettings();
7444 // Ensure we have actually reconfigured twice
7445 // The rate settings should have been set again even though
7446 // they haven't changed.
7447 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007448 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007449
7450 video_stream_encoder_->Stop();
7451}
7452
philipeld9cc8c02019-09-16 14:53:40 +02007453struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007454 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7455 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7456 MOCK_METHOD(void,
7457 RequestEncoderSwitch,
7458 (const webrtc::SdpVideoFormat& format),
7459 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007460};
7461
philipel9b058032020-02-10 11:30:00 +01007462TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7463 constexpr int kDontCare = 100;
7464 StrictMock<MockEncoderSelector> encoder_selector;
7465 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7466 &fake_encoder_, &encoder_selector);
7467 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7468
7469 // Reset encoder for new configuration to take effect.
7470 ConfigureEncoder(video_encoder_config_.Copy());
7471
7472 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7473
7474 video_source_.IncomingCapturedFrame(
7475 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007476 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007477 video_stream_encoder_->Stop();
7478
7479 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7480 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007481 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7482 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007483 video_stream_encoder_.reset();
7484}
7485
7486TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7487 constexpr int kDontCare = 100;
7488
7489 NiceMock<MockEncoderSelector> encoder_selector;
7490 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7491 video_send_config_.encoder_settings.encoder_switch_request_callback =
7492 &switch_callback;
7493 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7494 &fake_encoder_, &encoder_selector);
7495 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7496
7497 // Reset encoder for new configuration to take effect.
7498 ConfigureEncoder(video_encoder_config_.Copy());
7499
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007500 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007501 .WillByDefault(Return(SdpVideoFormat("AV1")));
7502 EXPECT_CALL(switch_callback,
7503 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7504 Field(&SdpVideoFormat::name, "AV1"))));
7505
Henrik Boström381d1092020-05-12 18:49:07 +02007506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007507 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7508 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7509 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007510 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007511 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007512 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007513 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007514
7515 video_stream_encoder_->Stop();
7516}
7517
7518TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7519 constexpr int kSufficientBitrateToNotDrop = 1000;
7520 constexpr int kDontCare = 100;
7521
7522 NiceMock<MockVideoEncoder> video_encoder;
7523 NiceMock<MockEncoderSelector> encoder_selector;
7524 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7525 video_send_config_.encoder_settings.encoder_switch_request_callback =
7526 &switch_callback;
7527 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7528 &video_encoder, &encoder_selector);
7529 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7530
7531 // Reset encoder for new configuration to take effect.
7532 ConfigureEncoder(video_encoder_config_.Copy());
7533
7534 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7535 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7536 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007537 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007538 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7539 /*stable_target_bitrate=*/
7540 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7541 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007542 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007543 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007544 /*cwnd_reduce_ratio=*/0);
7545
7546 ON_CALL(video_encoder, Encode(_, _))
7547 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7548 ON_CALL(encoder_selector, OnEncoderBroken())
7549 .WillByDefault(Return(SdpVideoFormat("AV2")));
7550
7551 rtc::Event encode_attempted;
7552 EXPECT_CALL(switch_callback,
7553 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7554 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7555 EXPECT_EQ(format.name, "AV2");
7556 encode_attempted.Set();
7557 });
7558
7559 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7560 encode_attempted.Wait(3000);
7561
Markus Handell28c71802021-11-08 10:11:55 +01007562 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007563
philipel9b058032020-02-10 11:30:00 +01007564 video_stream_encoder_->Stop();
7565
7566 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7567 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007568 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7569 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007570 video_stream_encoder_.reset();
7571}
7572
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007573TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007574 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007575 const int kFrameWidth = 320;
7576 const int kFrameHeight = 180;
7577
7578 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007579 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007580 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007581 /*target_bitrate=*/rate,
7582 /*stable_target_bitrate=*/rate,
7583 /*link_allocation=*/rate,
7584 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007585 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007586 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007587
7588 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007589 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007590 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7591 frame.set_rotation(kVideoRotation_270);
7592 video_source_.IncomingCapturedFrame(frame);
7593 WaitForEncodedFrame(timestamp_ms);
7594 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7595
7596 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007597 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007599 /*target_bitrate=*/new_stable_rate,
7600 /*stable_target_bitrate=*/new_stable_rate,
7601 /*link_allocation=*/rate,
7602 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007603 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007604 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007605 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7606 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7607 video_stream_encoder_->Stop();
7608}
7609
7610TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007611 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007612 const int kFrameWidth = 320;
7613 const int kFrameHeight = 180;
7614
7615 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007616 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007617 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007618 /*target_bitrate=*/rate,
7619 /*stable_target_bitrate=*/rate,
7620 /*link_allocation=*/rate,
7621 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007622 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007623 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007624
7625 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007626 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007627 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7628 frame.set_rotation(kVideoRotation_270);
7629 video_source_.IncomingCapturedFrame(frame);
7630 WaitForEncodedFrame(timestamp_ms);
7631 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7632
7633 // Set a higher target rate without changing the link_allocation. Should not
7634 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007635 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007637 /*target_bitrate=*/rate,
7638 /*stable_target_bitrate=*/new_stable_rate,
7639 /*link_allocation=*/rate,
7640 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007641 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007642 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007643 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7644 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7645 video_stream_encoder_->Stop();
7646}
7647
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007648TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7649 test::ScopedFieldTrials field_trials(
7650 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7651 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7652 const int kFramerateFps = 30;
7653 const int kWidth = 1920;
7654 const int kHeight = 1080;
7655 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7656 // Works on screenshare mode.
7657 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7658 // We rely on the automatic resolution adaptation, but we handle framerate
7659 // adaptation manually by mocking the stats proxy.
7660 video_source_.set_adaptation_enabled(true);
7661
7662 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007664 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007665 video_stream_encoder_->SetSource(&video_source_,
7666 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007667 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007668
7669 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7670 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7671
7672 // Pass enough frames with the full update to trigger animation detection.
7673 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007674 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007675 frame.set_ntp_time_ms(timestamp_ms);
7676 frame.set_timestamp_us(timestamp_ms * 1000);
7677 video_source_.IncomingCapturedFrame(frame);
7678 WaitForEncodedFrame(timestamp_ms);
7679 }
7680
7681 // Resolution should be limited.
7682 rtc::VideoSinkWants expected;
7683 expected.max_framerate_fps = kFramerateFps;
7684 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007685 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007686
7687 // Pass one frame with no known update.
7688 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007689 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007690 frame.set_ntp_time_ms(timestamp_ms);
7691 frame.set_timestamp_us(timestamp_ms * 1000);
7692 frame.clear_update_rect();
7693
7694 video_source_.IncomingCapturedFrame(frame);
7695 WaitForEncodedFrame(timestamp_ms);
7696
7697 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007698 EXPECT_THAT(video_source_.sink_wants(),
7699 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007700
7701 video_stream_encoder_->Stop();
7702}
7703
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007704TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7705 const int kWidth = 720; // 540p adapted down.
7706 const int kHeight = 405;
7707 const int kNumFrames = 3;
7708 // Works on screenshare mode.
7709 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7710 /*num_spatial_layers=*/2, /*screenshare=*/true);
7711
7712 video_source_.set_adaptation_enabled(true);
7713
7714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007716
7717 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7718
7719 // Pass enough frames with the full update to trigger animation detection.
7720 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007721 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007722 frame.set_ntp_time_ms(timestamp_ms);
7723 frame.set_timestamp_us(timestamp_ms * 1000);
7724 video_source_.IncomingCapturedFrame(frame);
7725 WaitForEncodedFrame(timestamp_ms);
7726 }
7727
7728 video_stream_encoder_->Stop();
7729}
7730
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007731TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7732 const float downscale_factors[] = {4.0, 2.0, 1.0};
7733 const int number_layers =
7734 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7735 VideoEncoderConfig config;
7736 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7737 for (int i = 0; i < number_layers; ++i) {
7738 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7739 config.simulcast_layers[i].active = true;
7740 }
7741 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007742 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007743 "VP8", /*max qp*/ 56, /*screencast*/ false,
7744 /*screenshare enabled*/ false);
7745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007746 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7747 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007748
7749 // First initialization.
7750 // Encoder should be initialized. Next frame should be key frame.
7751 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7752 sink_.SetNumExpectedLayers(number_layers);
7753 int64_t timestamp_ms = kFrameIntervalMs;
7754 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7755 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007756 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007757 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7758 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7759 VideoFrameType::kVideoFrameKey,
7760 VideoFrameType::kVideoFrameKey}));
7761
7762 // Disable top layer.
7763 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7764 config.simulcast_layers[number_layers - 1].active = false;
7765 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7766 sink_.SetNumExpectedLayers(number_layers - 1);
7767 timestamp_ms += kFrameIntervalMs;
7768 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7769 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007770 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007771 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7772 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7773 VideoFrameType::kVideoFrameDelta,
7774 VideoFrameType::kVideoFrameDelta}));
7775
7776 // Re-enable top layer.
7777 // Encoder should be re-initialized. Next frame should be key frame.
7778 config.simulcast_layers[number_layers - 1].active = true;
7779 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7780 sink_.SetNumExpectedLayers(number_layers);
7781 timestamp_ms += kFrameIntervalMs;
7782 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7783 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007784 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007785 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7786 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7787 VideoFrameType::kVideoFrameKey,
7788 VideoFrameType::kVideoFrameKey}));
7789
7790 // Top layer max rate change.
7791 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7792 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7793 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7794 sink_.SetNumExpectedLayers(number_layers);
7795 timestamp_ms += kFrameIntervalMs;
7796 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7797 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007798 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007799 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7800 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7801 VideoFrameType::kVideoFrameDelta,
7802 VideoFrameType::kVideoFrameDelta}));
7803
7804 // Top layer resolution change.
7805 // Encoder should be re-initialized. Next frame should be key frame.
7806 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7807 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7808 sink_.SetNumExpectedLayers(number_layers);
7809 timestamp_ms += kFrameIntervalMs;
7810 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7811 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007812 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007813 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7814 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7815 VideoFrameType::kVideoFrameKey,
7816 VideoFrameType::kVideoFrameKey}));
7817 video_stream_encoder_->Stop();
7818}
7819
Henrik Boström1124ed12021-02-25 10:30:39 +01007820TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7821 const int kFrameWidth = 1280;
7822 const int kFrameHeight = 720;
7823
7824 SetUp();
7825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007826 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007827
7828 // Capturing a frame should reconfigure the encoder and expose the encoder
7829 // resolution, which is the same as the input frame.
7830 int64_t timestamp_ms = kFrameIntervalMs;
7831 video_source_.IncomingCapturedFrame(
7832 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7833 WaitForEncodedFrame(timestamp_ms);
7834 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7835 EXPECT_THAT(video_source_.sink_wants().resolutions,
7836 ::testing::ElementsAreArray(
7837 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7838
7839 video_stream_encoder_->Stop();
7840}
7841
7842TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7843 // Pick downscale factors such that we never encode at full resolution - this
7844 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007845 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007846 // encoder should not ask for the frame resolution. This allows video frames
7847 // to have the appearence of one resolution but optimize its internal buffers
7848 // for what is actually encoded.
7849 const size_t kNumSimulcastLayers = 3u;
7850 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7851 const int kFrameWidth = 1280;
7852 const int kFrameHeight = 720;
7853 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7854 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7855 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7856 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7857 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7858 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7859
7860 VideoEncoderConfig config;
7861 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7862 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7863 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7864 config.simulcast_layers[i].active = true;
7865 }
7866 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007867 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007868 "VP8", /*max qp*/ 56, /*screencast*/ false,
7869 /*screenshare enabled*/ false);
7870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007871 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7872 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007873
7874 // Capture a frame with all layers active.
7875 int64_t timestamp_ms = kFrameIntervalMs;
7876 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7877 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7878 video_source_.IncomingCapturedFrame(
7879 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7880 WaitForEncodedFrame(timestamp_ms);
7881 // Expect encoded resolutions to match the expected simulcast layers.
7882 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7883 EXPECT_THAT(
7884 video_source_.sink_wants().resolutions,
7885 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7886
7887 // Capture a frame with one of the layers inactive.
7888 timestamp_ms += kFrameIntervalMs;
7889 config.simulcast_layers[2].active = false;
7890 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7891 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7892 video_source_.IncomingCapturedFrame(
7893 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7894 WaitForEncodedFrame(timestamp_ms);
7895
7896 // Expect encoded resolutions to match the expected simulcast layers.
7897 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7898 EXPECT_THAT(video_source_.sink_wants().resolutions,
7899 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7900
7901 // Capture a frame with all but one layer turned off.
7902 timestamp_ms += kFrameIntervalMs;
7903 config.simulcast_layers[1].active = false;
7904 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7905 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7906 video_source_.IncomingCapturedFrame(
7907 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7908 WaitForEncodedFrame(timestamp_ms);
7909
7910 // Expect encoded resolutions to match the expected simulcast layers.
7911 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7912 EXPECT_THAT(video_source_.sink_wants().resolutions,
7913 ::testing::ElementsAreArray({kLayer0Size}));
7914
7915 video_stream_encoder_->Stop();
7916}
7917
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007918TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007919 ResetEncoder("VP8", 1, 1, 1, false);
7920
Niels Möller8b692902021-06-14 12:04:57 +02007921 // Force encoder reconfig.
7922 video_source_.IncomingCapturedFrame(
7923 CreateFrame(1, codec_width_, codec_height_));
7924 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7925
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007926 // Set QP on encoded frame and pass the frame to encode complete callback.
7927 // Since QP is present QP parsing won't be triggered and the original value
7928 // should be kept.
7929 EncodedImage encoded_image;
7930 encoded_image.qp_ = 123;
7931 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7932 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7933 CodecSpecificInfo codec_info;
7934 codec_info.codecType = kVideoCodecVP8;
7935 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7936 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7937 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7938 video_stream_encoder_->Stop();
7939}
7940
7941TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007942 ResetEncoder("VP8", 1, 1, 1, false);
7943
Niels Möller8b692902021-06-14 12:04:57 +02007944 // Force encoder reconfig.
7945 video_source_.IncomingCapturedFrame(
7946 CreateFrame(1, codec_width_, codec_height_));
7947 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7948
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007949 // Pass an encoded frame without QP to encode complete callback. QP should be
7950 // parsed and set.
7951 EncodedImage encoded_image;
7952 encoded_image.qp_ = -1;
7953 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7954 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7955 CodecSpecificInfo codec_info;
7956 codec_info.codecType = kVideoCodecVP8;
7957 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7958 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7959 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7960 video_stream_encoder_->Stop();
7961}
7962
7963TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7964 webrtc::test::ScopedFieldTrials field_trials(
7965 "WebRTC-QpParsingKillSwitch/Enabled/");
7966
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007967 ResetEncoder("VP8", 1, 1, 1, false);
7968
Niels Möller8b692902021-06-14 12:04:57 +02007969 // Force encoder reconfig.
7970 video_source_.IncomingCapturedFrame(
7971 CreateFrame(1, codec_width_, codec_height_));
7972 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7973
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007974 EncodedImage encoded_image;
7975 encoded_image.qp_ = -1;
7976 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7977 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7978 CodecSpecificInfo codec_info;
7979 codec_info.codecType = kVideoCodecVP8;
7980 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7981 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7982 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7983 video_stream_encoder_->Stop();
7984}
7985
Sergey Silkind19e3b92021-03-16 10:05:30 +00007986TEST_F(VideoStreamEncoderTest,
7987 QualityScalingNotAllowed_QualityScalingDisabled) {
7988 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7989
7990 // Disable scaling settings in encoder info.
7991 fake_encoder_.SetQualityScaling(false);
7992 // Disable quality scaling in encoder config.
7993 video_encoder_config.is_quality_scaling_allowed = false;
7994 ConfigureEncoder(std::move(video_encoder_config));
7995
7996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007997 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007998
7999 test::FrameForwarder source;
8000 video_stream_encoder_->SetSource(
8001 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8002 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8003 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8004
8005 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8006 WaitForEncodedFrame(1);
8007 video_stream_encoder_->TriggerQualityLow();
8008 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8009
8010 video_stream_encoder_->Stop();
8011}
8012
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008013TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8014 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8015
8016 // Disable scaling settings in encoder info.
8017 fake_encoder_.SetQualityScaling(false);
8018 // Set QP trusted in encoder info.
8019 fake_encoder_.SetIsQpTrusted(true);
8020 // Enable quality scaling in encoder config.
8021 video_encoder_config.is_quality_scaling_allowed = false;
8022 ConfigureEncoder(std::move(video_encoder_config));
8023
8024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008025 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008026
8027 test::FrameForwarder source;
8028 video_stream_encoder_->SetSource(
8029 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8030 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8031 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8032
8033 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8034 WaitForEncodedFrame(1);
8035 video_stream_encoder_->TriggerQualityLow();
8036 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8037
8038 video_stream_encoder_->Stop();
8039}
8040
Shuhai Pengf2707702021-09-29 17:19:44 +08008041TEST_F(VideoStreamEncoderTest,
8042 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8043 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8044
8045 // Disable scaling settings in encoder info.
8046 fake_encoder_.SetQualityScaling(false);
8047 // Set QP trusted in encoder info.
8048 fake_encoder_.SetIsQpTrusted(true);
8049 // Enable quality scaling in encoder config.
8050 video_encoder_config.is_quality_scaling_allowed = false;
8051 ConfigureEncoder(std::move(video_encoder_config));
8052
8053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008054 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008055
8056 test::FrameForwarder source;
8057 video_stream_encoder_->SetSource(
8058 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8059 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8060 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8061
8062 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8063 WaitForEncodedFrame(1);
8064 video_stream_encoder_->TriggerQualityLow();
8065 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8066
8067 video_stream_encoder_->Stop();
8068}
8069
8070TEST_F(VideoStreamEncoderTest,
8071 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8072 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8073
8074 // Disable scaling settings in encoder info.
8075 fake_encoder_.SetQualityScaling(false);
8076 // Set QP trusted in encoder info.
8077 fake_encoder_.SetIsQpTrusted(false);
8078 // Enable quality scaling in encoder config.
8079 video_encoder_config.is_quality_scaling_allowed = false;
8080 ConfigureEncoder(std::move(video_encoder_config));
8081
8082 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008083 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008084
8085 test::FrameForwarder source;
8086 video_stream_encoder_->SetSource(
8087 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8088 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8089 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8090
8091 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8092 WaitForEncodedFrame(1);
8093 video_stream_encoder_->TriggerQualityLow();
8094 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8095
8096 video_stream_encoder_->Stop();
8097}
8098
8099TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8100 // Set QP trusted in encoder info.
8101 fake_encoder_.SetIsQpTrusted(false);
8102
8103 const int MinEncBitrateKbps = 30;
8104 const int MaxEncBitrateKbps = 100;
8105 const int MinStartBitrateKbp = 50;
8106 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8107 /*frame_size_pixels=*/codec_width_ * codec_height_,
8108 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8109 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8110 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8111
8112 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008113 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008114
8115 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8116
8117 VideoEncoderConfig video_encoder_config;
8118 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8119 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8120 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8121 MinEncBitrateKbps * 1000;
8122 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8123 kMaxPayloadLength);
8124
8125 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8126 WaitForEncodedFrame(1);
8127 EXPECT_EQ(
8128 MaxEncBitrateKbps,
8129 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8130 EXPECT_EQ(
8131 MinEncBitrateKbps,
8132 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8133
8134 video_stream_encoder_->Stop();
8135}
8136
8137TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8138 // Set QP trusted in encoder info.
8139 fake_encoder_.SetIsQpTrusted(false);
8140
8141 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8142 EncoderInfoSettings::
8143 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8144 codec_width_ * codec_height_,
8145 EncoderInfoSettings::
8146 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8147 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8148
8149 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8150 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8151 const int TargetEncBitrate = MaxEncBitrate;
8152 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8153 DataRate::BitsPerSec(TargetEncBitrate),
8154 DataRate::BitsPerSec(TargetEncBitrate),
8155 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8156
8157 VideoEncoderConfig video_encoder_config;
8158 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8159 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8160 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8161 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8162 kMaxPayloadLength);
8163
8164 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8165 WaitForEncodedFrame(1);
8166 EXPECT_EQ(
8167 MaxEncBitrate / 1000,
8168 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8169 EXPECT_EQ(
8170 MinEncBitrate / 1000,
8171 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8172
8173 video_stream_encoder_->Stop();
8174}
8175
Sergey Silkind19e3b92021-03-16 10:05:30 +00008176#if !defined(WEBRTC_IOS)
8177// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8178// disabled by default on iOS.
8179TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8180 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8181
8182 // Disable scaling settings in encoder info.
8183 fake_encoder_.SetQualityScaling(false);
8184 // Enable quality scaling in encoder config.
8185 video_encoder_config.is_quality_scaling_allowed = true;
8186 ConfigureEncoder(std::move(video_encoder_config));
8187
8188 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008189 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008190
8191 test::FrameForwarder source;
8192 video_stream_encoder_->SetSource(
8193 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8194 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8195 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8196
8197 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8198 WaitForEncodedFrame(1);
8199 video_stream_encoder_->TriggerQualityLow();
8200 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8201
8202 video_stream_encoder_->Stop();
8203}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008204
8205TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8206 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8207
8208 // Disable scaling settings in encoder info.
8209 fake_encoder_.SetQualityScaling(false);
8210 // Set QP trusted in encoder info.
8211 fake_encoder_.SetIsQpTrusted(true);
8212 // Enable quality scaling in encoder config.
8213 video_encoder_config.is_quality_scaling_allowed = true;
8214 ConfigureEncoder(std::move(video_encoder_config));
8215
8216 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008217 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008218
8219 test::FrameForwarder source;
8220 video_stream_encoder_->SetSource(
8221 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8222 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8223 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8224
8225 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8226 WaitForEncodedFrame(1);
8227 video_stream_encoder_->TriggerQualityLow();
8228 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8229
8230 video_stream_encoder_->Stop();
8231}
Shuhai Pengf2707702021-09-29 17:19:44 +08008232
8233TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8234 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8235
8236 // Disable scaling settings in encoder info.
8237 fake_encoder_.SetQualityScaling(false);
8238 // Set QP not trusted in encoder info.
8239 fake_encoder_.SetIsQpTrusted(false);
8240 // Enable quality scaling in encoder config.
8241 video_encoder_config.is_quality_scaling_allowed = true;
8242 ConfigureEncoder(std::move(video_encoder_config));
8243
8244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008245 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008246
8247 test::FrameForwarder source;
8248 video_stream_encoder_->SetSource(
8249 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8250 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8252
8253 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8254 WaitForEncodedFrame(1);
8255 video_stream_encoder_->TriggerQualityLow();
8256 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8257 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8258 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8259
8260 video_stream_encoder_->Stop();
8261}
8262
8263TEST_F(VideoStreamEncoderTest,
8264 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8265 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8266
8267 // Disable scaling settings in encoder info.
8268 fake_encoder_.SetQualityScaling(false);
8269 // Set QP trusted in encoder info.
8270 fake_encoder_.SetIsQpTrusted(true);
8271 // Enable quality scaling in encoder config.
8272 video_encoder_config.is_quality_scaling_allowed = true;
8273 ConfigureEncoder(std::move(video_encoder_config));
8274
8275 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008276 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008277
8278 test::FrameForwarder source;
8279 video_stream_encoder_->SetSource(
8280 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8281 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8282 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8283
8284 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8285 WaitForEncodedFrame(1);
8286 video_stream_encoder_->TriggerQualityLow();
8287 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8288 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8289
8290 video_stream_encoder_->Stop();
8291}
8292
8293TEST_F(VideoStreamEncoderTest,
8294 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8295 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8296
8297 // Disable scaling settings in encoder info.
8298 fake_encoder_.SetQualityScaling(false);
8299 // Set QP trusted in encoder info.
8300 fake_encoder_.SetIsQpTrusted(false);
8301 // Enable quality scaling in encoder config.
8302 video_encoder_config.is_quality_scaling_allowed = true;
8303 ConfigureEncoder(std::move(video_encoder_config));
8304
8305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008306 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008307
8308 test::FrameForwarder source;
8309 video_stream_encoder_->SetSource(
8310 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8311 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8312 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8313
8314 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8315 WaitForEncodedFrame(1);
8316 video_stream_encoder_->TriggerQualityLow();
8317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8318
8319 video_stream_encoder_->Stop();
8320}
8321
Sergey Silkind19e3b92021-03-16 10:05:30 +00008322#endif
8323
Henrik Boström56db9ff2021-03-24 09:06:45 +01008324// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8325class VideoStreamEncoderWithRealEncoderTest
8326 : public VideoStreamEncoderTest,
8327 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8328 public:
8329 VideoStreamEncoderWithRealEncoderTest()
8330 : VideoStreamEncoderTest(),
8331 codec_type_(std::get<0>(GetParam())),
8332 allow_i420_conversion_(std::get<1>(GetParam())) {}
8333
8334 void SetUp() override {
8335 VideoStreamEncoderTest::SetUp();
8336 std::unique_ptr<VideoEncoder> encoder;
8337 switch (codec_type_) {
8338 case kVideoCodecVP8:
8339 encoder = VP8Encoder::Create();
8340 break;
8341 case kVideoCodecVP9:
8342 encoder = VP9Encoder::Create();
8343 break;
8344 case kVideoCodecAV1:
8345 encoder = CreateLibaomAv1Encoder();
8346 break;
8347 case kVideoCodecH264:
8348 encoder =
8349 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8350 break;
8351 case kVideoCodecMultiplex:
8352 mock_encoder_factory_for_multiplex_ =
8353 std::make_unique<MockVideoEncoderFactory>();
8354 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8355 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8356 .WillRepeatedly([] { return VP8Encoder::Create(); });
8357 encoder = std::make_unique<MultiplexEncoderAdapter>(
8358 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8359 false);
8360 break;
8361 default:
Artem Titovd3251962021-11-15 16:57:07 +01008362 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008363 }
8364 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8365 }
8366
8367 void TearDown() override {
8368 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008369 // Ensure `video_stream_encoder_` is destroyed before
8370 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008371 video_stream_encoder_.reset();
8372 VideoStreamEncoderTest::TearDown();
8373 }
8374
8375 protected:
8376 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8377 std::unique_ptr<VideoEncoder> encoder) {
8378 // Configure VSE to use the encoder.
8379 encoder_ = std::move(encoder);
8380 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8381 encoder_.get(), &encoder_selector_);
8382 video_send_config_.encoder_settings.encoder_factory =
8383 encoder_proxy_factory_.get();
8384 VideoEncoderConfig video_encoder_config;
8385 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8386 video_encoder_config_ = video_encoder_config.Copy();
8387 ConfigureEncoder(video_encoder_config_.Copy());
8388
8389 // Set bitrate to ensure frame is not dropped.
8390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008391 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008392 }
8393
8394 const VideoCodecType codec_type_;
8395 const bool allow_i420_conversion_;
8396 NiceMock<MockEncoderSelector> encoder_selector_;
8397 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8398 std::unique_ptr<VideoEncoder> encoder_;
8399 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8400};
8401
8402TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8403 auto native_i420_frame = test::CreateMappableNativeFrame(
8404 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8405 video_source_.IncomingCapturedFrame(native_i420_frame);
8406 WaitForEncodedFrame(codec_width_, codec_height_);
8407
8408 auto mappable_native_buffer =
8409 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8410 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8411 mappable_native_buffer->GetMappedFramedBuffers();
8412 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8413 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8414 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8415 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8416}
8417
8418TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8419 auto native_nv12_frame = test::CreateMappableNativeFrame(
8420 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8421 video_source_.IncomingCapturedFrame(native_nv12_frame);
8422 WaitForEncodedFrame(codec_width_, codec_height_);
8423
8424 auto mappable_native_buffer =
8425 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8426 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8427 mappable_native_buffer->GetMappedFramedBuffers();
8428 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8429 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8430 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8431 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8432
8433 if (!allow_i420_conversion_) {
8434 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8435 }
8436}
8437
Erik Språng7444b192021-06-02 14:02:13 +02008438TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8439 if (codec_type_ == kVideoCodecMultiplex) {
8440 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8441 return;
8442 }
8443
8444 const size_t kNumSpatialLayers = 3u;
8445 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8446 const int kFrameWidth = 1280;
8447 const int kFrameHeight = 720;
8448 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8449 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8450 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8451 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8452 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8453 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8454
8455 VideoEncoderConfig config;
8456 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8457 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008458 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008459 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8460 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8461 vp9_settings.numberOfTemporalLayers = 3;
8462 vp9_settings.automaticResizeOn = false;
8463 config.encoder_specific_settings =
8464 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8465 vp9_settings);
8466 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8467 /*fps=*/30.0,
8468 /*first_active_layer=*/0,
8469 /*num_spatial_layers=*/3,
8470 /*num_temporal_layers=*/3,
8471 /*is_screenshare=*/false);
8472 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8473 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008474 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008475 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8476 /*fps=*/30.0,
8477 /*first_active_layer=*/0,
8478 /*num_spatial_layers=*/3,
8479 /*num_temporal_layers=*/3,
8480 /*is_screenshare=*/false);
8481 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8482 } else {
8483 // Simulcast for VP8/H264.
8484 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8485 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8486 config.simulcast_layers[i].scale_resolution_down_by =
8487 kDownscaleFactors[i];
8488 config.simulcast_layers[i].active = true;
8489 }
8490 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8491 // Turn off frame dropping to prevent flakiness.
8492 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8493 h264_settings.frameDroppingOn = false;
8494 config.encoder_specific_settings = rtc::make_ref_counted<
8495 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8496 }
8497 }
8498
8499 auto set_layer_active = [&](int layer_idx, bool active) {
8500 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8501 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8502 config.spatial_layers[layer_idx].active = active;
8503 } else {
8504 config.simulcast_layers[layer_idx].active = active;
8505 }
8506 };
8507
8508 config.video_stream_factory =
8509 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8510 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8511 /*screencast*/ false,
8512 /*screenshare enabled*/ false);
8513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008514 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8515 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008516
8517 // Capture a frame with all layers active.
8518 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8519 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8520 int64_t timestamp_ms = kFrameIntervalMs;
8521 video_source_.IncomingCapturedFrame(
8522 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8523
8524 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8525 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8526
8527 // Capture a frame with one of the layers inactive.
8528 set_layer_active(2, false);
8529 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8530 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8531 timestamp_ms += kFrameIntervalMs;
8532 video_source_.IncomingCapturedFrame(
8533 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8534 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8535
8536 // New target bitrates signaled based on lower resolution.
8537 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8539 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8540 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8541
8542 // Re-enable the top layer.
8543 set_layer_active(2, true);
8544 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8545 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8546 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8547
8548 // Bitrate target adjusted back up to enable HD layer...
8549 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8550 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8551 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8552 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8553
8554 // ...then add a new frame.
8555 timestamp_ms += kFrameIntervalMs;
8556 video_source_.IncomingCapturedFrame(
8557 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8558 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8559 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8560
8561 video_stream_encoder_->Stop();
8562}
8563
Henrik Boström56db9ff2021-03-24 09:06:45 +01008564std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8565 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8566 VideoCodecType codec_type = std::get<0>(info.param);
8567 bool allow_i420_conversion = std::get<1>(info.param);
8568 std::string str;
8569 switch (codec_type) {
8570 case kVideoCodecGeneric:
8571 str = "Generic";
8572 break;
8573 case kVideoCodecVP8:
8574 str = "VP8";
8575 break;
8576 case kVideoCodecVP9:
8577 str = "VP9";
8578 break;
8579 case kVideoCodecAV1:
8580 str = "AV1";
8581 break;
8582 case kVideoCodecH264:
8583 str = "H264";
8584 break;
8585 case kVideoCodecMultiplex:
8586 str = "Multiplex";
8587 break;
8588 default:
Artem Titovd3251962021-11-15 16:57:07 +01008589 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008590 }
8591 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8592 return str;
8593}
8594
8595constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8596 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8597constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8598 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8599constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8600 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8601constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8602 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8603#if defined(WEBRTC_USE_H264)
8604constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8605 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8606
8607// The windows compiler does not tolerate #if statements inside the
8608// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8609// and without H264).
8610INSTANTIATE_TEST_SUITE_P(
8611 All,
8612 VideoStreamEncoderWithRealEncoderTest,
8613 ::testing::Values(kVP8DisallowConversion,
8614 kVP9DisallowConversion,
8615 kAV1AllowConversion,
8616 kMultiplexDisallowConversion,
8617 kH264AllowConversion),
8618 TestParametersVideoCodecAndAllowI420ConversionToString);
8619#else
8620INSTANTIATE_TEST_SUITE_P(
8621 All,
8622 VideoStreamEncoderWithRealEncoderTest,
8623 ::testing::Values(kVP8DisallowConversion,
8624 kVP9DisallowConversion,
8625 kAV1AllowConversion,
8626 kMultiplexDisallowConversion),
8627 TestParametersVideoCodecAndAllowI420ConversionToString);
8628#endif
8629
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008630class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8631 protected:
8632 void RunTest(const std::vector<VideoStream>& configs,
8633 const int expected_num_init_encode) {
8634 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008635 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008636 InsertFrameAndWaitForEncoded();
8637 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8638 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008639 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8640 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008641
8642 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8643 ConfigureEncoder(configs[1]);
8644 InsertFrameAndWaitForEncoded();
8645 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8646 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008647 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008648 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008649 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008650
8651 video_stream_encoder_->Stop();
8652 }
8653
8654 void ConfigureEncoder(const VideoStream& stream) {
8655 VideoEncoderConfig config;
8656 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8657 config.max_bitrate_bps = stream.max_bitrate_bps;
8658 config.simulcast_layers[0] = stream;
8659 config.video_stream_factory =
8660 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8661 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8662 /*conference_mode=*/false);
8663 video_stream_encoder_->ConfigureEncoder(std::move(config),
8664 kMaxPayloadLength);
8665 }
8666
8667 void OnBitrateUpdated(DataRate bitrate) {
8668 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8669 bitrate, bitrate, bitrate, 0, 0, 0);
8670 }
8671
8672 void InsertFrameAndWaitForEncoded() {
8673 timestamp_ms_ += kFrameIntervalMs;
8674 video_source_.IncomingCapturedFrame(
8675 CreateFrame(timestamp_ms_, kWidth, kHeight));
8676 sink_.WaitForEncodedFrame(timestamp_ms_);
8677 }
8678
8679 void ExpectEqual(const VideoCodec& actual,
8680 const VideoStream& expected) const {
8681 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8682 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8683 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8684 static_cast<unsigned int>(expected.min_bitrate_bps));
8685 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8686 static_cast<unsigned int>(expected.max_bitrate_bps));
8687 EXPECT_EQ(actual.simulcastStream[0].width,
8688 kWidth / expected.scale_resolution_down_by);
8689 EXPECT_EQ(actual.simulcastStream[0].height,
8690 kHeight / expected.scale_resolution_down_by);
8691 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8692 expected.num_temporal_layers);
8693 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8694 }
8695
8696 VideoStream DefaultConfig() const {
8697 VideoStream stream;
8698 stream.max_framerate = 25;
8699 stream.min_bitrate_bps = 35000;
8700 stream.max_bitrate_bps = 900000;
8701 stream.scale_resolution_down_by = 1.0;
8702 stream.num_temporal_layers = 1;
8703 stream.bitrate_priority = 1.0;
8704 stream.scalability_mode = "";
8705 return stream;
8706 }
8707
8708 const int kWidth = 640;
8709 const int kHeight = 360;
8710 int64_t timestamp_ms_ = 0;
8711};
8712
8713TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8714 VideoStream config1 = DefaultConfig();
8715 VideoStream config2 = config1;
8716 config2.max_framerate++;
8717
8718 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8719}
8720
8721TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8722 VideoStream config1 = DefaultConfig();
8723 VideoStream config2 = config1;
8724 config2.min_bitrate_bps += 10000;
8725
8726 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8727}
8728
8729TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8730 VideoStream config1 = DefaultConfig();
8731 VideoStream config2 = config1;
8732 config2.max_bitrate_bps += 100000;
8733
8734 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8735}
8736
8737TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8738 VideoStream config1 = DefaultConfig();
8739 VideoStream config2 = config1;
8740 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8741
8742 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8743}
8744
8745TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8746 VideoStream config1 = DefaultConfig();
8747 VideoStream config2 = config1;
8748 config2.scale_resolution_down_by *= 2;
8749
8750 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8751}
8752
8753TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8754 VideoStream config1 = DefaultConfig();
8755 VideoStream config2 = config1;
8756 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8757
8758 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8759}
8760
8761TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8762 VideoStream config1 = DefaultConfig();
8763 VideoStream config2 = config1;
8764 config2.scalability_mode = "L1T2";
8765
8766 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8767}
8768
Markus Handellb4e96d42021-11-05 12:00:55 +01008769TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8770 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8771 auto* adapter_ptr = adapter.get();
8772 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01008773 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8774 nullptr;
8775 EXPECT_CALL(*adapter_ptr, Initialize)
8776 .WillOnce(Invoke([&video_stream_encoder_callback](
8777 FrameCadenceAdapterInterface::Callback* callback) {
8778 video_stream_encoder_callback = callback;
8779 }));
8780 TaskQueueBase* encoder_queue = nullptr;
8781 auto video_stream_encoder =
8782 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01008783
Markus Handell8d87c462021-12-16 11:37:16 +01008784 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Not(Eq(absl::nullopt))));
Markus Handellb4e96d42021-11-05 12:00:55 +01008785 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01008786 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01008787 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8788 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008789 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01008790 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008791 Mock::VerifyAndClearExpectations(adapter_ptr);
8792
Markus Handell8d87c462021-12-16 11:37:16 +01008793 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01008794 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01008795 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01008796 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8797 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008798 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01008799 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008800}
8801
8802TEST(VideoStreamEncoderFrameCadenceTest,
8803 ForwardsFramesIntoFrameCadenceAdapter) {
8804 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8805 auto* adapter_ptr = adapter.get();
8806 test::FrameForwarder video_source;
8807 SimpleVideoStreamEncoderFactory factory;
8808 auto video_stream_encoder = factory.Create(std::move(adapter));
8809 video_stream_encoder->SetSource(
8810 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8811
8812 EXPECT_CALL(*adapter_ptr, OnFrame);
8813 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8814 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01008815 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01008816}
8817
Markus Handellee225432021-11-29 12:35:12 +01008818TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8819 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8820 auto* adapter_ptr = adapter.get();
8821 test::FrameForwarder video_source;
8822 SimpleVideoStreamEncoderFactory factory;
8823 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8824 nullptr;
8825 EXPECT_CALL(*adapter_ptr, Initialize)
8826 .WillOnce(Invoke([&video_stream_encoder_callback](
8827 FrameCadenceAdapterInterface::Callback* callback) {
8828 video_stream_encoder_callback = callback;
8829 }));
8830 TaskQueueBase* encoder_queue = nullptr;
8831 auto video_stream_encoder =
8832 factory.Create(std::move(adapter), &encoder_queue);
8833
8834 // This is just to make the VSE operational. We'll feed a frame directly by
8835 // the callback interface.
8836 video_stream_encoder->SetSource(
8837 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8838
8839 VideoEncoderConfig video_encoder_config;
8840 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
8841 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
8842 /*max_data_payload_length=*/1000);
8843
8844 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
8845 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01008846 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01008847 factory.DepleteTaskQueues();
8848}
8849
Markus Handell8d87c462021-12-16 11:37:16 +01008850TEST(VideoStreamEncoderFrameCadenceTest,
8851 DeactivatesActivatesLayersOnBitrateChanges) {
8852 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8853 auto* adapter_ptr = adapter.get();
8854 SimpleVideoStreamEncoderFactory factory;
8855 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8856 nullptr;
8857 EXPECT_CALL(*adapter_ptr, Initialize)
8858 .WillOnce(Invoke([&video_stream_encoder_callback](
8859 FrameCadenceAdapterInterface::Callback* callback) {
8860 video_stream_encoder_callback = callback;
8861 }));
8862 TaskQueueBase* encoder_queue = nullptr;
8863 auto video_stream_encoder =
8864 factory.Create(std::move(adapter), &encoder_queue);
8865
8866 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
8867 // {150000, 450000}.
8868 VideoEncoderConfig video_encoder_config;
8869 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8870 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8871 kMaxPayloadLength);
8872 // Ensure an encoder is created.
8873 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8874
8875 // Both layers enabled at 1 MBit/s.
8876 video_stream_encoder->OnBitrateUpdated(
8877 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8878 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8879 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8880 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8881 factory.DepleteTaskQueues();
8882 Mock::VerifyAndClearExpectations(adapter_ptr);
8883
8884 // Layer 1 disabled at 200 KBit/s.
8885 video_stream_encoder->OnBitrateUpdated(
8886 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
8887 DataRate::KilobitsPerSec(200), 0, 0, 0);
8888 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8889 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8890 factory.DepleteTaskQueues();
8891 Mock::VerifyAndClearExpectations(adapter_ptr);
8892
8893 // All layers off at suspended video.
8894 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
8895 DataRate::Zero(), 0, 0, 0);
8896 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
8897 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8898 factory.DepleteTaskQueues();
8899 Mock::VerifyAndClearExpectations(adapter_ptr);
8900
8901 // Both layers enabled again back at 1 MBit/s.
8902 video_stream_encoder->OnBitrateUpdated(
8903 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8904 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8905 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8906 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8907 factory.DepleteTaskQueues();
8908}
8909
8910TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
8911 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8912 auto* adapter_ptr = adapter.get();
8913 SimpleVideoStreamEncoderFactory factory;
8914 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8915 nullptr;
8916 EXPECT_CALL(*adapter_ptr, Initialize)
8917 .WillOnce(Invoke([&video_stream_encoder_callback](
8918 FrameCadenceAdapterInterface::Callback* callback) {
8919 video_stream_encoder_callback = callback;
8920 }));
8921 TaskQueueBase* encoder_queue = nullptr;
8922 auto video_stream_encoder =
8923 factory.Create(std::move(adapter), &encoder_queue);
8924
8925 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
8926 VideoEncoderConfig video_encoder_config;
8927 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8928 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8929 kMaxPayloadLength);
8930 video_stream_encoder->OnBitrateUpdated(
8931 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8932 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8933
8934 // Pass a frame which has unconverged results.
8935 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8936 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8937 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8938 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8939 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
8940 CodecSpecificInfo codec_specific;
8941 codec_specific.codecType = kVideoCodecGeneric;
8942 return codec_specific;
8943 }));
8944 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
8945 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8946 factory.DepleteTaskQueues();
8947 Mock::VerifyAndClearExpectations(adapter_ptr);
8948 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8949
8950 // Pass a frame which converges in layer 0 and not in layer 1.
8951 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
8952 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8953 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8954 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8955 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
8956 CodecSpecificInfo codec_specific;
8957 codec_specific.codecType = kVideoCodecGeneric;
8958 return codec_specific;
8959 }));
8960 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
8961 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8962 factory.DepleteTaskQueues();
8963 Mock::VerifyAndClearExpectations(adapter_ptr);
8964 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8965}
8966
Markus Handell2e0f4f02021-12-21 19:14:58 +01008967TEST(VideoStreamEncoderFrameCadenceTest,
8968 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
8969 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8970 auto* adapter_ptr = adapter.get();
8971 MockVideoSourceInterface mock_source;
8972 SimpleVideoStreamEncoderFactory factory;
8973 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8974 nullptr;
8975 EXPECT_CALL(*adapter_ptr, Initialize)
8976 .WillOnce(Invoke([&video_stream_encoder_callback](
8977 FrameCadenceAdapterInterface::Callback* callback) {
8978 video_stream_encoder_callback = callback;
8979 }));
8980 TaskQueueBase* encoder_queue = nullptr;
8981 auto video_stream_encoder =
8982 factory.Create(std::move(adapter), &encoder_queue);
8983 video_stream_encoder->SetSource(
8984 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8985 VideoEncoderConfig config;
8986 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8987 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
8988 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
8989 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
8990 // Ensure the encoder is set up.
8991 factory.DepleteTaskQueues();
8992
8993 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest).WillOnce(Return(true));
8994 EXPECT_CALL(mock_source, RequestRefreshFrame);
8995 video_stream_encoder->SendKeyFrame();
8996 factory.DepleteTaskQueues();
8997 Mock::VerifyAndClearExpectations(adapter_ptr);
8998 Mock::VerifyAndClearExpectations(&mock_source);
8999
9000 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest).WillOnce(Return(false));
9001 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9002 video_stream_encoder->SendKeyFrame();
9003 factory.DepleteTaskQueues();
9004}
9005
perkj26091b12016-09-01 01:17:40 -07009006} // namespace webrtc