blob: d143a639c591f7fdf21e1161ef800a1869e20d8a [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010021#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020022#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010023#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010024#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080025#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020027#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020028#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010029#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010030#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020031#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020032#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010033#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020034#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010035#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020036#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070037#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020039#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010040#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
41#include "modules/video_coding/codecs/h264/include/h264.h"
42#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
43#include "modules/video_coding/codecs/vp8/include/vp8.h"
44#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020045#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020046#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020047#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010048#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020049#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010050#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020051#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020052#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080053#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020054#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010055#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020056#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020057#include "test/encoder_settings.h"
58#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020059#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010060#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020061#include "test/gmock.h"
62#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010063#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020064#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020065#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010066#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020067#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070068
69namespace webrtc {
70
sprang57c2fff2017-01-16 06:24:02 -080071using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020072using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020073using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020075using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020076using ::testing::Ge;
77using ::testing::Gt;
78using ::testing::Le;
79using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010080using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010081using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010082using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010083using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010085using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020086using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080087
perkj803d97f2016-11-01 11:45:46 -070088namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020089const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010090const int kQpLow = 1;
91const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020092const int kMinFramerateFps = 2;
93const int kMinBalancedFramerateFps = 7;
94const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080095const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020096const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
97const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
98const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
99const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800100const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700101const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200102const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200103const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200104const VideoEncoder::ResolutionBitrateLimits
105 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800108
Asa Persson606d3cb2021-10-04 10:07:11 +0200109uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200110 0x00, 0x00, 0x03, 0x03, 0xF4,
111 0x05, 0x03, 0xC7, 0xE0, 0x1B,
112 0x41, 0x10, 0x8D, 0x00};
113
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100114const uint8_t kCodedFrameVp8Qp25[] = {
115 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
116 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
117 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
118
perkj803d97f2016-11-01 11:45:46 -0700119class TestBuffer : public webrtc::I420Buffer {
120 public:
121 TestBuffer(rtc::Event* event, int width, int height)
122 : I420Buffer(width, height), event_(event) {}
123
124 private:
125 friend class rtc::RefCountedObject<TestBuffer>;
126 ~TestBuffer() override {
127 if (event_)
128 event_->Set();
129 }
130 rtc::Event* const event_;
131};
132
Henrik Boström56db9ff2021-03-24 09:06:45 +0100133// A fake native buffer that can't be converted to I420. Upon scaling, it
134// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700135class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
136 public:
137 FakeNativeBuffer(rtc::Event* event, int width, int height)
138 : event_(event), width_(width), height_(height) {}
139 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
140 int width() const override { return width_; }
141 int height() const override { return height_; }
142 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
143 return nullptr;
144 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100145 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
146 int offset_x,
147 int offset_y,
148 int crop_width,
149 int crop_height,
150 int scaled_width,
151 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200152 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
153 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100154 }
Noah Richards51db4212019-06-12 06:59:12 -0700155
156 private:
157 friend class rtc::RefCountedObject<FakeNativeBuffer>;
158 ~FakeNativeBuffer() override {
159 if (event_)
160 event_->Set();
161 }
162 rtc::Event* const event_;
163 const int width_;
164 const int height_;
165};
166
Evan Shrubsole895556e2020-10-05 09:15:13 +0200167// A fake native buffer that is backed by an NV12 buffer.
168class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
169 public:
170 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
171 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
172
173 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
174 int width() const override { return nv12_buffer_->width(); }
175 int height() const override { return nv12_buffer_->height(); }
176 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
177 return nv12_buffer_->ToI420();
178 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200179 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
180 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
181 if (absl::c_find(types, Type::kNV12) != types.end()) {
182 return nv12_buffer_;
183 }
184 return nullptr;
185 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200186 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
187
188 private:
189 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
190 ~FakeNV12NativeBuffer() override {
191 if (event_)
192 event_->Set();
193 }
194 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
195 rtc::Event* const event_;
196};
197
Niels Möller7dc26b72017-12-06 10:27:48 +0100198class CpuOveruseDetectorProxy : public OveruseFrameDetector {
199 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200200 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
201 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200202 last_target_framerate_fps_(-1),
203 framerate_updated_event_(true /* manual_reset */,
204 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 virtual ~CpuOveruseDetectorProxy() {}
206
207 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200208 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100209 last_target_framerate_fps_ = framerate_fps;
210 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200211 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 }
213
214 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200215 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100216 return last_target_framerate_fps_;
217 }
218
Niels Möller4db138e2018-04-19 09:04:13 +0200219 CpuOveruseOptions GetOptions() { return options_; }
220
Henrik Boström381d1092020-05-12 18:49:07 +0200221 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
222
Niels Möller7dc26b72017-12-06 10:27:48 +0100223 private:
Markus Handella3765182020-07-08 13:13:32 +0200224 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100225 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200226 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100227};
228
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200229class FakeVideoSourceRestrictionsListener
230 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200232 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200233 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200234 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200235 RTC_DCHECK(was_restrictions_updated_);
236 }
237
238 rtc::Event* restrictions_updated_event() {
239 return &restrictions_updated_event_;
240 }
241
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200242 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200243 void OnVideoSourceRestrictionsUpdated(
244 VideoSourceRestrictions restrictions,
245 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200246 rtc::scoped_refptr<Resource> reason,
247 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200248 was_restrictions_updated_ = true;
249 restrictions_updated_event_.Set();
250 }
251
252 private:
253 bool was_restrictions_updated_;
254 rtc::Event restrictions_updated_event_;
255};
256
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200257auto WantsFps(Matcher<int> fps_matcher) {
258 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
259 fps_matcher);
260}
261
262auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
263 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
264 AllOf(max_pixel_matcher, Gt(0)));
265}
266
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200267auto ResolutionMax() {
268 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200269 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200270 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
271 Eq(absl::nullopt)));
272}
273
274auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200287 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200288}
289
290auto FpsMaxResolutionMax() {
291 return AllOf(FpsMax(), ResolutionMax());
292}
293
294auto UnlimitedSinkWants() {
295 return AllOf(FpsUnlimited(), ResolutionMax());
296}
297
298auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
299 Matcher<int> fps_range_matcher;
300
301 if (last_frame_pixels <= 320 * 240) {
302 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200303 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304 fps_range_matcher = AllOf(Ge(10), Le(15));
305 } else if (last_frame_pixels <= 640 * 480) {
306 fps_range_matcher = Ge(15);
307 } else {
308 fps_range_matcher = Eq(kDefaultFramerate);
309 }
310 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
311 fps_range_matcher);
312}
313
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200314auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
315 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
316 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
321}
322
323auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
325}
326
327auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
328 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
329 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
330}
331
332auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
333 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
334 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
335}
336
337auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
338 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
339 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
340}
341
342auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
343 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
344 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
345}
346
mflodmancc3d4422017-08-03 08:27:51 -0700347class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700348 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200349 VideoStreamEncoderUnderTest(TimeController* time_controller,
350 TaskQueueFactory* task_queue_factory,
351 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100352 const VideoStreamEncoderSettings& settings,
353 VideoStreamEncoder::BitrateAllocationCallbackType
354 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200355 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100356 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200357 stats_proxy,
358 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200359 std::unique_ptr<OveruseFrameDetector>(
360 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100361 new CpuOveruseDetectorProxy(stats_proxy)),
Markus Handellb4e96d42021-11-05 12:00:55 +0100362 FrameCadenceAdapterInterface::Create(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100363 task_queue_factory,
Markus Handell2b10c472021-10-28 15:29:42 +0200364 TaskQueueBase::Current(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100365 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200366 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200367 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200368 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200369 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200370 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200371 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200372 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100374 }
perkj803d97f2016-11-01 11:45:46 -0700375
Henrik Boström381d1092020-05-12 18:49:07 +0200376 void SetSourceAndWaitForRestrictionsUpdated(
377 rtc::VideoSourceInterface<VideoFrame>* source,
378 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200379 FakeVideoSourceRestrictionsListener listener;
380 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200381 SetSource(source, degradation_preference);
382 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200383 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200384 }
385
386 void SetSourceAndWaitForFramerateUpdated(
387 rtc::VideoSourceInterface<VideoFrame>* source,
388 const DegradationPreference& degradation_preference) {
389 overuse_detector_proxy_->framerate_updated_event()->Reset();
390 SetSource(source, degradation_preference);
391 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
392 }
393
394 void OnBitrateUpdatedAndWaitForManagedResources(
395 DataRate target_bitrate,
396 DataRate stable_target_bitrate,
397 DataRate link_allocation,
398 uint8_t fraction_lost,
399 int64_t round_trip_time_ms,
400 double cwnd_reduce_ratio) {
401 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
402 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
403 // Bitrate is updated on the encoder queue.
404 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200405 }
406
kthelgason2fc52542017-03-03 00:24:41 -0800407 // This is used as a synchronisation mechanism, to make sure that the
408 // encoder queue is not blocked before we start sending it frames.
409 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100410 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200411 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800412 ASSERT_TRUE(event.Wait(5000));
413 }
414
Henrik Boström91aa7322020-04-28 12:24:33 +0200415 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200416 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200418 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200419 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 event.Set();
421 });
422 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200423 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200424 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200425
Henrik Boström91aa7322020-04-28 12:24:33 +0200426 void TriggerCpuUnderuse() {
427 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200428 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200429 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200430 event.Set();
431 });
432 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200433 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200434 }
kthelgason876222f2016-11-29 01:44:11 -0800435
Henrik Boström91aa7322020-04-28 12:24:33 +0200436 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200437 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200438 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200439 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200440 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 event.Set();
442 });
443 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200444 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200445 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200446 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200447 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200448 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200449 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200450 event.Set();
451 });
452 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200453 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200454 }
455
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200456 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100457 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200458 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
459 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200460 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700461};
462
Noah Richards51db4212019-06-12 06:59:12 -0700463// Simulates simulcast behavior and makes highest stream resolutions divisible
464// by 4.
465class CroppingVideoStreamFactory
466 : public VideoEncoderConfig::VideoStreamFactoryInterface {
467 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200468 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700469
470 private:
471 std::vector<VideoStream> CreateEncoderStreams(
472 int width,
473 int height,
474 const VideoEncoderConfig& encoder_config) override {
475 std::vector<VideoStream> streams = test::CreateVideoStreams(
476 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700477 return streams;
478 }
Noah Richards51db4212019-06-12 06:59:12 -0700479};
480
sprangb1ca0732017-02-01 08:38:12 -0800481class AdaptingFrameForwarder : public test::FrameForwarder {
482 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200483 explicit AdaptingFrameForwarder(TimeController* time_controller)
484 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700485 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800486
487 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200488 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800489 adaptation_enabled_ = enabled;
490 }
491
asaperssonfab67072017-04-04 05:51:49 -0700492 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200493 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800494 return adaptation_enabled_;
495 }
496
Henrik Boström1124ed12021-02-25 10:30:39 +0100497 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
498 // the resolution or frame rate was different than it is currently. If
499 // something else is modified, such as encoder resolutions, but the resolution
500 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700501 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200502 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700503 return last_wants_;
504 }
505
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200506 absl::optional<int> last_sent_width() const { return last_width_; }
507 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800508
sprangb1ca0732017-02-01 08:38:12 -0800509 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200510 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
511 time_controller_->AdvanceTime(TimeDelta::Millis(0));
512
sprangb1ca0732017-02-01 08:38:12 -0800513 int cropped_width = 0;
514 int cropped_height = 0;
515 int out_width = 0;
516 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700517 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000518 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
519 << "w=" << video_frame.width()
520 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700521 if (adapter_.AdaptFrameResolution(
522 video_frame.width(), video_frame.height(),
523 video_frame.timestamp_us() * 1000, &cropped_width,
524 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100525 VideoFrame adapted_frame =
526 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200527 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100528 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200529 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100530 .set_timestamp_ms(99)
531 .set_rotation(kVideoRotation_0)
532 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100533 if (video_frame.has_update_rect()) {
534 adapted_frame.set_update_rect(
535 video_frame.update_rect().ScaleWithFrame(
536 video_frame.width(), video_frame.height(), 0, 0,
537 video_frame.width(), video_frame.height(), out_width,
538 out_height));
539 }
sprangc5d62e22017-04-02 23:53:04 -0700540 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800541 last_width_.emplace(adapted_frame.width());
542 last_height_.emplace(adapted_frame.height());
543 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200544 last_width_ = absl::nullopt;
545 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700546 }
sprangb1ca0732017-02-01 08:38:12 -0800547 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000548 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800549 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800550 last_width_.emplace(video_frame.width());
551 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800552 }
553 }
554
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200555 void OnOutputFormatRequest(int width, int height) {
556 absl::optional<std::pair<int, int>> target_aspect_ratio =
557 std::make_pair(width, height);
558 absl::optional<int> max_pixel_count = width * height;
559 absl::optional<int> max_fps;
560 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
561 max_fps);
562 }
563
sprangb1ca0732017-02-01 08:38:12 -0800564 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
565 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200566 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100567 rtc::VideoSinkWants prev_wants = sink_wants_locked();
568 bool did_adapt =
569 prev_wants.max_pixel_count != wants.max_pixel_count ||
570 prev_wants.target_pixel_count != wants.target_pixel_count ||
571 prev_wants.max_framerate_fps != wants.max_framerate_fps;
572 if (did_adapt) {
573 last_wants_ = prev_wants;
574 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100575 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200576 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800577 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200578
579 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800580 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200581 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
582 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200583 absl::optional<int> last_width_;
584 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800585};
sprangc5d62e22017-04-02 23:53:04 -0700586
Niels Möller213618e2018-07-24 09:29:58 +0200587// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700588class MockableSendStatisticsProxy : public SendStatisticsProxy {
589 public:
590 MockableSendStatisticsProxy(Clock* clock,
591 const VideoSendStream::Config& config,
592 VideoEncoderConfig::ContentType content_type)
593 : SendStatisticsProxy(clock, config, content_type) {}
594
595 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200596 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700597 if (mock_stats_)
598 return *mock_stats_;
599 return SendStatisticsProxy::GetStats();
600 }
601
Niels Möller213618e2018-07-24 09:29:58 +0200602 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200603 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200604 if (mock_stats_)
605 return mock_stats_->input_frame_rate;
606 return SendStatisticsProxy::GetInputFrameRate();
607 }
sprangc5d62e22017-04-02 23:53:04 -0700608 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200609 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700610 mock_stats_.emplace(stats);
611 }
612
613 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200614 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700615 mock_stats_.reset();
616 }
617
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200618 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
619 on_frame_dropped_ = std::move(callback);
620 }
621
sprangc5d62e22017-04-02 23:53:04 -0700622 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200623 void OnFrameDropped(DropReason reason) override {
624 SendStatisticsProxy::OnFrameDropped(reason);
625 if (on_frame_dropped_)
626 on_frame_dropped_(reason);
627 }
628
Markus Handella3765182020-07-08 13:13:32 +0200629 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200630 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200631 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700632};
633
Markus Handellb4e96d42021-11-05 12:00:55 +0100634class SimpleVideoStreamEncoderFactory {
635 public:
636 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
637 public:
638 using VideoStreamEncoder::VideoStreamEncoder;
639 ~AdaptedVideoStreamEncoder() { Stop(); }
640 };
641
642 SimpleVideoStreamEncoderFactory()
643 : time_controller_(Timestamp::Millis(0)),
644 task_queue_factory_(time_controller_.CreateTaskQueueFactory()),
645 stats_proxy_(std::make_unique<MockableSendStatisticsProxy>(
646 time_controller_.GetClock(),
647 VideoSendStream::Config(nullptr),
648 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
649 encoder_settings_(
650 VideoEncoder::Capabilities(/*loss_notification=*/false)),
651 fake_encoder_(time_controller_.GetClock()),
652 encoder_factory_(&fake_encoder_) {
653 encoder_settings_.encoder_factory = &encoder_factory_;
654 }
655
656 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
657 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter) {
658 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
659 time_controller_.GetClock(),
660 /*number_of_cores=*/1,
661 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
662 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
663 std::move(zero_hertz_adapter), task_queue_factory_.get(),
664 TaskQueueBase::Current(),
665 VideoStreamEncoder::BitrateAllocationCallbackType::
666 kVideoBitrateAllocation);
667 result->SetSink(&sink_, /*rotation_applied=*/false);
668 return result;
669 }
670
671 private:
672 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
673 public:
674 ~NullEncoderSink() override = default;
675 void OnEncoderConfigurationChanged(
676 std::vector<VideoStream> streams,
677 bool is_svc,
678 VideoEncoderConfig::ContentType content_type,
679 int min_transmit_bitrate_bps) override {}
680 void OnBitrateAllocationUpdated(
681 const VideoBitrateAllocation& allocation) override {}
682 void OnVideoLayersAllocationUpdated(
683 VideoLayersAllocation allocation) override {}
684 Result OnEncodedImage(
685 const EncodedImage& encoded_image,
686 const CodecSpecificInfo* codec_specific_info) override {
687 return Result(EncodedImageCallback::Result::OK);
688 }
689 };
690
691 GlobalSimulatedTimeController time_controller_;
692 std::unique_ptr<TaskQueueFactory> task_queue_factory_;
693 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
694 VideoStreamEncoderSettings encoder_settings_;
695 test::FakeEncoder fake_encoder_;
696 test::VideoEncoderProxyFactory encoder_factory_;
697 NullEncoderSink sink_;
698};
699
700class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
701 public:
702 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
703 MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override));
704 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
705};
706
philipel9b058032020-02-10 11:30:00 +0100707class MockEncoderSelector
708 : public VideoEncoderFactory::EncoderSelectorInterface {
709 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200710 MOCK_METHOD(void,
711 OnCurrentEncoder,
712 (const SdpVideoFormat& format),
713 (override));
714 MOCK_METHOD(absl::optional<SdpVideoFormat>,
715 OnAvailableBitrate,
716 (const DataRate& rate),
717 (override));
718 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100719};
720
perkj803d97f2016-11-01 11:45:46 -0700721} // namespace
722
mflodmancc3d4422017-08-03 08:27:51 -0700723class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700724 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200725 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700726
mflodmancc3d4422017-08-03 08:27:51 -0700727 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700728 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700729 codec_width_(320),
730 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200731 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200732 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200733 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700734 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200735 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700736 video_send_config_,
737 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200738 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700739
740 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700741 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700742 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200743 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800744 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200745 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200746 video_send_config_.rtp.payload_name = "FAKE";
747 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700748
Per512ecb32016-09-23 15:52:06 +0200749 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200750 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200751 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
752 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
753 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100754 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700755
Niels Möllerf1338562018-04-26 09:51:47 +0200756 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800757 }
758
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100759 void ConfigureEncoder(
760 VideoEncoderConfig video_encoder_config,
761 VideoStreamEncoder::BitrateAllocationCallbackType
762 allocation_callback_type =
763 VideoStreamEncoder::BitrateAllocationCallbackType::
764 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700765 if (video_stream_encoder_)
766 video_stream_encoder_->Stop();
767 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200768 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100769 video_send_config_.encoder_settings, allocation_callback_type));
Asa Persson606d3cb2021-10-04 10:07:11 +0200770 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700771 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700772 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200773 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700774 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200775 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800777 }
778
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100779 void ResetEncoder(const std::string& payload_name,
780 size_t num_streams,
781 size_t num_temporal_layers,
782 unsigned char num_spatial_layers,
783 bool screenshare,
784 VideoStreamEncoder::BitrateAllocationCallbackType
785 allocation_callback_type =
786 VideoStreamEncoder::BitrateAllocationCallbackType::
787 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200788 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800789
790 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200791 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
792 num_streams, &video_encoder_config);
793 for (auto& layer : video_encoder_config.simulcast_layers) {
794 layer.num_temporal_layers = num_temporal_layers;
795 layer.max_framerate = kDefaultFramerate;
796 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100797 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200798 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700799 video_encoder_config.content_type =
800 screenshare ? VideoEncoderConfig::ContentType::kScreen
801 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700802 if (payload_name == "VP9") {
803 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
804 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200805 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700806 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200807 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
808 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700809 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100810 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700811 }
812
sprang57c2fff2017-01-16 06:24:02 -0800813 VideoFrame CreateFrame(int64_t ntp_time_ms,
814 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200815 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200816 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200817 destruction_event, codec_width_, codec_height_))
818 .set_ntp_time_ms(ntp_time_ms)
819 .set_timestamp_ms(99)
820 .set_rotation(kVideoRotation_0)
821 .build();
perkj26091b12016-09-01 01:17:40 -0700822 }
823
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100824 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
825 rtc::Event* destruction_event,
826 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200827 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200828 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200829 destruction_event, codec_width_, codec_height_))
830 .set_ntp_time_ms(ntp_time_ms)
831 .set_timestamp_ms(99)
832 .set_rotation(kVideoRotation_0)
833 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
834 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100835 }
836
sprang57c2fff2017-01-16 06:24:02 -0800837 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200838 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
839 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200840 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200841 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200842 .set_ntp_time_ms(ntp_time_ms)
843 .set_timestamp_ms(ntp_time_ms)
844 .set_rotation(kVideoRotation_0)
845 .build();
perkj803d97f2016-11-01 11:45:46 -0700846 }
847
Evan Shrubsole895556e2020-10-05 09:15:13 +0200848 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200849 return VideoFrame::Builder()
850 .set_video_frame_buffer(NV12Buffer::Create(width, height))
851 .set_ntp_time_ms(ntp_time_ms)
852 .set_timestamp_ms(ntp_time_ms)
853 .set_rotation(kVideoRotation_0)
854 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200855 }
856
Noah Richards51db4212019-06-12 06:59:12 -0700857 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
858 rtc::Event* destruction_event,
859 int width,
860 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200861 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200862 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200863 destruction_event, width, height))
864 .set_ntp_time_ms(ntp_time_ms)
865 .set_timestamp_ms(99)
866 .set_rotation(kVideoRotation_0)
867 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700868 }
869
Evan Shrubsole895556e2020-10-05 09:15:13 +0200870 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
871 rtc::Event* destruction_event,
872 int width,
873 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200874 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200875 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200876 destruction_event, width, height))
877 .set_ntp_time_ms(ntp_time_ms)
878 .set_timestamp_ms(99)
879 .set_rotation(kVideoRotation_0)
880 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200881 }
882
Noah Richards51db4212019-06-12 06:59:12 -0700883 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
884 rtc::Event* destruction_event) const {
885 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
886 codec_height_);
887 }
888
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100889 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200891 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100892
893 video_source_.IncomingCapturedFrame(
894 CreateFrame(1, codec_width_, codec_height_));
895 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200896 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100897 }
898
sprang4847ae62017-06-27 07:06:52 -0700899 void WaitForEncodedFrame(int64_t expected_ntp_time) {
900 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200901 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700902 }
903
904 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
905 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200906 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700907 return ok;
908 }
909
910 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
911 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200912 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700913 }
914
915 void ExpectDroppedFrame() {
916 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200917 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700918 }
919
920 bool WaitForFrame(int64_t timeout_ms) {
921 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200922 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700923 return ok;
924 }
925
perkj26091b12016-09-01 01:17:40 -0700926 class TestEncoder : public test::FakeEncoder {
927 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200928 explicit TestEncoder(TimeController* time_controller)
929 : FakeEncoder(time_controller->GetClock()),
930 time_controller_(time_controller) {
931 RTC_DCHECK(time_controller_);
932 }
perkj26091b12016-09-01 01:17:40 -0700933
perkjfa10b552016-10-02 23:45:26 -0700934 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700936 block_next_encode_ = true;
937 }
938
Erik Språngaed30702018-11-05 12:57:17 +0100939 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200940 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200941 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100942 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100943 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100944 info.scaling_settings = VideoEncoder::ScalingSettings(
945 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100946 }
947 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100948 for (int i = 0; i < kMaxSpatialLayers; ++i) {
949 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100950 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100951 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100952 for (int tid = 0; tid < num_layers; ++tid)
953 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100954 }
955 }
Erik Språngaed30702018-11-05 12:57:17 +0100956 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200957
958 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100959 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200960 info.apply_alignment_to_all_simulcast_layers =
961 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200962 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800963 if (is_qp_trusted_.has_value()) {
964 info.is_qp_trusted = is_qp_trusted_;
965 }
Erik Språngaed30702018-11-05 12:57:17 +0100966 return info;
kthelgason876222f2016-11-29 01:44:11 -0800967 }
968
Erik Språngb7cb7b52019-02-26 15:52:33 +0100969 int32_t RegisterEncodeCompleteCallback(
970 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200971 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100972 encoded_image_callback_ = callback;
973 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
974 }
975
perkjfa10b552016-10-02 23:45:26 -0700976 void ContinueEncode() { continue_encode_event_.Set(); }
977
978 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
979 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200980 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700981 EXPECT_EQ(timestamp_, timestamp);
982 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
983 }
984
kthelgason2fc52542017-03-03 00:24:41 -0800985 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200986 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800987 quality_scaling_ = b;
988 }
kthelgasonad9010c2017-02-14 00:46:51 -0800989
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100990 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200991 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100992 requested_resolution_alignment_ = requested_resolution_alignment;
993 }
994
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200995 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
996 MutexLock lock(&local_mutex_);
997 apply_alignment_to_all_simulcast_layers_ = b;
998 }
999
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001000 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001001 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001002 is_hardware_accelerated_ = is_hardware_accelerated;
1003 }
1004
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001005 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1006 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001007 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001008 temporal_layers_supported_[spatial_idx] = supported;
1009 }
1010
Sergey Silkin6456e352019-07-08 17:56:40 +02001011 void SetResolutionBitrateLimits(
1012 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001013 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001014 resolution_bitrate_limits_ = thresholds;
1015 }
1016
sprangfe627f32017-03-29 08:24:59 -07001017 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001018 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001019 force_init_encode_failed_ = force_failure;
1020 }
1021
Niels Möller6bb5ab92019-01-11 11:11:10 +01001022 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001023 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001024 rate_factor_ = rate_factor;
1025 }
1026
Erik Språngd7329ca2019-02-21 21:19:53 +01001027 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001028 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001029 return last_framerate_;
1030 }
1031
Erik Språngd7329ca2019-02-21 21:19:53 +01001032 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001033 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001034 return last_update_rect_;
1035 }
1036
Niels Möller87e2d782019-03-07 10:18:23 +01001037 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001038 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001039 return last_frame_types_;
1040 }
1041
1042 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001043 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001044 keyframe ? VideoFrameType::kVideoFrameKey
1045 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001046 {
Markus Handella3765182020-07-08 13:13:32 +02001047 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001048 last_frame_types_ = frame_type;
1049 }
Niels Möllerb859b322019-03-07 12:40:01 +01001050 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001051 }
1052
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001053 void InjectEncodedImage(const EncodedImage& image,
1054 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001055 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001056 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001057 }
1058
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001059 void SetEncodedImageData(
1060 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001061 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001062 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001063 }
1064
Erik Språngd7329ca2019-02-21 21:19:53 +01001065 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001066 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001067 expect_null_frame_ = true;
1068 }
1069
Erik Språng5056af02019-09-02 15:53:11 +02001070 absl::optional<VideoEncoder::RateControlParameters>
1071 GetAndResetLastRateControlSettings() {
1072 auto settings = last_rate_control_settings_;
1073 last_rate_control_settings_.reset();
1074 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001075 }
1076
Henrik Boström56db9ff2021-03-24 09:06:45 +01001077 int GetLastInputWidth() const {
1078 MutexLock lock(&local_mutex_);
1079 return last_input_width_;
1080 }
1081
1082 int GetLastInputHeight() const {
1083 MutexLock lock(&local_mutex_);
1084 return last_input_height_;
1085 }
1086
Evan Shrubsole895556e2020-10-05 09:15:13 +02001087 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1088 MutexLock lock(&local_mutex_);
1089 return last_input_pixel_format_;
1090 }
1091
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001092 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001093 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001094 return num_set_rates_;
1095 }
1096
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001097 void SetPreferredPixelFormats(
1098 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1099 pixel_formats) {
1100 MutexLock lock(&local_mutex_);
1101 preferred_pixel_formats_ = std::move(pixel_formats);
1102 }
1103
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001104 void SetIsQpTrusted(absl::optional<bool> trusted) {
1105 MutexLock lock(&local_mutex_);
1106 is_qp_trusted_ = trusted;
1107 }
1108
perkjfa10b552016-10-02 23:45:26 -07001109 private:
perkj26091b12016-09-01 01:17:40 -07001110 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001111 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001112 bool block_encode;
1113 {
Markus Handella3765182020-07-08 13:13:32 +02001114 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001115 if (expect_null_frame_) {
1116 EXPECT_EQ(input_image.timestamp(), 0u);
1117 EXPECT_EQ(input_image.width(), 1);
1118 last_frame_types_ = *frame_types;
1119 expect_null_frame_ = false;
1120 } else {
1121 EXPECT_GT(input_image.timestamp(), timestamp_);
1122 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1123 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1124 }
perkj26091b12016-09-01 01:17:40 -07001125
1126 timestamp_ = input_image.timestamp();
1127 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001128 last_input_width_ = input_image.width();
1129 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001130 block_encode = block_next_encode_;
1131 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001132 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001134 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001135 }
Niels Möllerb859b322019-03-07 12:40:01 +01001136 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001137 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001138 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001139
perkj26091b12016-09-01 01:17:40 -07001140 return result;
1141 }
1142
Niels Möller08ae7ce2020-09-23 15:58:12 +02001143 CodecSpecificInfo EncodeHook(
1144 EncodedImage& encoded_image,
1145 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001146 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001147 {
1148 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001149 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001150 }
1151 MutexLock lock(&local_mutex_);
1152 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001153 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001154 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001155 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001156 }
1157
sprangfe627f32017-03-29 08:24:59 -07001158 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001159 const Settings& settings) override {
1160 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001161
Markus Handella3765182020-07-08 13:13:32 +02001162 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001163 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001164
Erik Språng82fad3d2018-03-21 09:57:23 +01001165 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001166 // Simulate setting up temporal layers, in order to validate the life
1167 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001168 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001169 frame_buffer_controller_ =
1170 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001171 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001172 if (force_init_encode_failed_) {
1173 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001174 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001175 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001176
Erik Språngb7cb7b52019-02-26 15:52:33 +01001177 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001178 return res;
1179 }
1180
Erik Språngb7cb7b52019-02-26 15:52:33 +01001181 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001182 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001183 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1184 initialized_ = EncoderState::kUninitialized;
1185 return FakeEncoder::Release();
1186 }
1187
Erik Språng16cb8f52019-04-12 13:59:09 +02001188 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001189 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001190 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001191 VideoBitrateAllocation adjusted_rate_allocation;
1192 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1193 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001194 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001195 adjusted_rate_allocation.SetBitrate(
1196 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001197 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001198 rate_factor_));
1199 }
1200 }
1201 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001202 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001203 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001204 RateControlParameters adjusted_paramters = parameters;
1205 adjusted_paramters.bitrate = adjusted_rate_allocation;
1206 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001207 }
1208
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001209 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001210 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001211 enum class EncoderState {
1212 kUninitialized,
1213 kInitializationFailed,
1214 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001215 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1216 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001217 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001218 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1219 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1220 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1221 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1222 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1223 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001224 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1225 false;
Markus Handella3765182020-07-08 13:13:32 +02001226 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001227 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1228 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001229 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001230 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001231 absl::optional<bool>
1232 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001233 local_mutex_);
1234 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1235 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1236 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001237 absl::optional<VideoEncoder::RateControlParameters>
1238 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001239 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1240 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001241 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001242 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001243 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1244 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001245 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001246 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001247 RTC_GUARDED_BY(local_mutex_);
1248 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001249 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1250 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001251 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1252 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001253 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001254 };
1255
mflodmancc3d4422017-08-03 08:27:51 -07001256 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001257 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001258 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1259 : time_controller_(time_controller), test_encoder_(test_encoder) {
1260 RTC_DCHECK(time_controller_);
1261 }
perkj26091b12016-09-01 01:17:40 -07001262
perkj26091b12016-09-01 01:17:40 -07001263 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001264 EXPECT_TRUE(
1265 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1266 }
1267
1268 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1269 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001270 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001271 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001272 return false;
perkj26091b12016-09-01 01:17:40 -07001273 {
Markus Handella3765182020-07-08 13:13:32 +02001274 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001275 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001276 }
1277 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001278 return true;
perkj26091b12016-09-01 01:17:40 -07001279 }
1280
sprangb1ca0732017-02-01 08:38:12 -08001281 void WaitForEncodedFrame(uint32_t expected_width,
1282 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001283 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001284 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001285 }
1286
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001287 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001288 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001289 uint32_t width = 0;
1290 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001291 {
Markus Handella3765182020-07-08 13:13:32 +02001292 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001293 width = last_width_;
1294 height = last_height_;
1295 }
1296 EXPECT_EQ(expected_height, height);
1297 EXPECT_EQ(expected_width, width);
1298 }
1299
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001300 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1301 VideoRotation rotation;
1302 {
Markus Handella3765182020-07-08 13:13:32 +02001303 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001304 rotation = last_rotation_;
1305 }
1306 EXPECT_EQ(expected_rotation, rotation);
1307 }
1308
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001309 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001310
sprangc5d62e22017-04-02 23:53:04 -07001311 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001312 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1313 bool ret = encoded_frame_event_.Wait(timeout_ms);
1314 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1315 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001316 }
1317
perkj26091b12016-09-01 01:17:40 -07001318 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001319 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001320 expect_frames_ = false;
1321 }
1322
asaperssonfab67072017-04-04 05:51:49 -07001323 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001324 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001325 return number_of_reconfigurations_;
1326 }
1327
asaperssonfab67072017-04-04 05:51:49 -07001328 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001329 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001330 return min_transmit_bitrate_bps_;
1331 }
1332
Erik Språngd7329ca2019-02-21 21:19:53 +01001333 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001334 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001335 num_expected_layers_ = num_layers;
1336 }
1337
Erik Språngb7cb7b52019-02-26 15:52:33 +01001338 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001339 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001340 return last_capture_time_ms_;
1341 }
1342
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001343 const EncodedImage& GetLastEncodedImage() {
1344 MutexLock lock(&mutex_);
1345 return last_encoded_image_;
1346 }
1347
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001348 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001349 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001350 return std::move(last_encoded_image_data_);
1351 }
1352
Per Kjellanderdcef6412020-10-07 15:09:05 +02001353 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1354 MutexLock lock(&mutex_);
1355 return last_bitrate_allocation_;
1356 }
1357
1358 int number_of_bitrate_allocations() const {
1359 MutexLock lock(&mutex_);
1360 return number_of_bitrate_allocations_;
1361 }
1362
Per Kjellandera9434842020-10-15 17:53:22 +02001363 VideoLayersAllocation GetLastVideoLayersAllocation() {
1364 MutexLock lock(&mutex_);
1365 return last_layers_allocation_;
1366 }
1367
1368 int number_of_layers_allocations() const {
1369 MutexLock lock(&mutex_);
1370 return number_of_layers_allocations_;
1371 }
1372
perkj26091b12016-09-01 01:17:40 -07001373 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001374 Result OnEncodedImage(
1375 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001376 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001377 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001378 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001379 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001380 last_encoded_image_data_ = std::vector<uint8_t>(
1381 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001382 uint32_t timestamp = encoded_image.Timestamp();
1383 if (last_timestamp_ != timestamp) {
1384 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001385 last_width_ = encoded_image._encodedWidth;
1386 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001387 } else {
1388 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001389 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1390 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001391 }
1392 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001393 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001394 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001395 if (num_received_layers_ == num_expected_layers_) {
1396 encoded_frame_event_.Set();
1397 }
sprangb1ca0732017-02-01 08:38:12 -08001398 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001399 }
1400
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001401 void OnEncoderConfigurationChanged(
1402 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001403 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001404 VideoEncoderConfig::ContentType content_type,
1405 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001406 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001407 ++number_of_reconfigurations_;
1408 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1409 }
1410
Per Kjellanderdcef6412020-10-07 15:09:05 +02001411 void OnBitrateAllocationUpdated(
1412 const VideoBitrateAllocation& allocation) override {
1413 MutexLock lock(&mutex_);
1414 ++number_of_bitrate_allocations_;
1415 last_bitrate_allocation_ = allocation;
1416 }
1417
Per Kjellandera9434842020-10-15 17:53:22 +02001418 void OnVideoLayersAllocationUpdated(
1419 VideoLayersAllocation allocation) override {
1420 MutexLock lock(&mutex_);
1421 ++number_of_layers_allocations_;
1422 last_layers_allocation_ = allocation;
1423 rtc::StringBuilder log;
1424 for (const auto& layer : allocation.active_spatial_layers) {
1425 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1426 << "[";
1427 for (const auto target_bitrate :
1428 layer.target_bitrate_per_temporal_layer) {
1429 log << target_bitrate.kbps() << ",";
1430 }
1431 log << "]";
1432 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001433 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001434 }
1435
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001436 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001437 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001438 TestEncoder* test_encoder_;
1439 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001440 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001441 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001442 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001443 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001444 uint32_t last_height_ = 0;
1445 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001446 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001447 size_t num_expected_layers_ = 1;
1448 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001449 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001450 int number_of_reconfigurations_ = 0;
1451 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001452 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1453 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001454 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1455 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001456 };
1457
Sergey Silkin5ee69672019-07-02 14:18:34 +02001458 class VideoBitrateAllocatorProxyFactory
1459 : public VideoBitrateAllocatorFactory {
1460 public:
1461 VideoBitrateAllocatorProxyFactory()
1462 : bitrate_allocator_factory_(
1463 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1464
1465 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1466 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001467 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001468 codec_config_ = codec;
1469 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1470 }
1471
1472 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001473 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001474 return codec_config_;
1475 }
1476
1477 private:
1478 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1479
Markus Handella3765182020-07-08 13:13:32 +02001480 mutable Mutex mutex_;
1481 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001482 };
1483
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001484 Clock* clock() { return time_controller_.GetClock(); }
1485 void AdvanceTime(TimeDelta duration) {
1486 time_controller_.AdvanceTime(duration);
1487 }
1488
1489 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1490
1491 protected:
1492 virtual TaskQueueFactory* GetTaskQueueFactory() {
1493 return time_controller_.GetTaskQueueFactory();
1494 }
1495
1496 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001497 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001498 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001499 int codec_width_;
1500 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001501 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001502 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001503 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001504 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001505 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001506 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001507 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001508 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001509};
1510
mflodmancc3d4422017-08-03 08:27:51 -07001511TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001514 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001515 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001516 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001517 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001518 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001519}
1520
mflodmancc3d4422017-08-03 08:27:51 -07001521TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001522 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001523 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001524 // The encoder will cache up to one frame for a short duration. Adding two
1525 // frames means that the first frame will be dropped and the second frame will
1526 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001527 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001528 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001529 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001530 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001531
Henrik Boström381d1092020-05-12 18:49:07 +02001532 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001533 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001534
Sebastian Janssona3177052018-04-10 13:05:49 +02001535 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001536 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001537 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1538
1539 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001540 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001541}
1542
mflodmancc3d4422017-08-03 08:27:51 -07001543TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001545 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001546 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001547 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001548
Henrik Boström381d1092020-05-12 18:49:07 +02001549 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001550 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1551
Sebastian Janssona3177052018-04-10 13:05:49 +02001552 // The encoder will cache up to one frame for a short duration. Adding two
1553 // frames means that the first frame will be dropped and the second frame will
1554 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001555 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001556 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001557
Henrik Boström381d1092020-05-12 18:49:07 +02001558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001559 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001560 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001561 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1562 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001563 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001564}
1565
mflodmancc3d4422017-08-03 08:27:51 -07001566TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001568 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001569 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001570 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001571
1572 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001573 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001574
perkja49cbd32016-09-16 07:53:41 -07001575 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001576 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001577 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001578}
1579
mflodmancc3d4422017-08-03 08:27:51 -07001580TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001582 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001583
perkja49cbd32016-09-16 07:53:41 -07001584 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001585 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001586
mflodmancc3d4422017-08-03 08:27:51 -07001587 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001588 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001589 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001590 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1591 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001592}
1593
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001594class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1595 public:
1596 VideoStreamEncoderBlockedTest() {}
1597
1598 TaskQueueFactory* GetTaskQueueFactory() override {
1599 return task_queue_factory_.get();
1600 }
1601
1602 private:
1603 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1604 CreateDefaultTaskQueueFactory();
1605};
1606
1607TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001608 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001609 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001610
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001611 int dropped_count = 0;
1612 stats_proxy_->SetDroppedFrameCallback(
1613 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1614 ++dropped_count;
1615 });
1616
perkj26091b12016-09-01 01:17:40 -07001617 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001618 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001619 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001620 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1621 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001622 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1623 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001624 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001625 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001626
mflodmancc3d4422017-08-03 08:27:51 -07001627 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001628
1629 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001630}
1631
Henrik Boström56db9ff2021-03-24 09:06:45 +01001632TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001634 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001635
1636 rtc::Event frame_destroyed_event;
1637 video_source_.IncomingCapturedFrame(
1638 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001639 WaitForEncodedFrame(1);
1640 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1641 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001642 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1643 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001644 video_stream_encoder_->Stop();
1645}
1646
Henrik Boström56db9ff2021-03-24 09:06:45 +01001647TEST_F(VideoStreamEncoderTest,
1648 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001649 // Use the cropping factory.
1650 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001651 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001652 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1653 kMaxPayloadLength);
1654 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1655
1656 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001658 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001659 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1660 WaitForEncodedFrame(1);
1661 // The encoder will have been configured once.
1662 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001663 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1664 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001665
1666 // Now send in a fake frame that needs to be cropped as the width/height
1667 // aren't divisible by 4 (see CreateEncoderStreams above).
1668 rtc::Event frame_destroyed_event;
1669 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1670 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001671 WaitForEncodedFrame(2);
1672 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1673 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001674 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1675 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001676 video_stream_encoder_->Stop();
1677}
1678
Evan Shrubsole895556e2020-10-05 09:15:13 +02001679TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1680 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001681 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001682
1683 video_source_.IncomingCapturedFrame(
1684 CreateNV12Frame(1, codec_width_, codec_height_));
1685 WaitForEncodedFrame(1);
1686 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1687 fake_encoder_.GetLastInputPixelFormat());
1688 video_stream_encoder_->Stop();
1689}
1690
Henrik Boström56db9ff2021-03-24 09:06:45 +01001691TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001692 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001693 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001694
1695 fake_encoder_.SetPreferredPixelFormats({});
1696
1697 rtc::Event frame_destroyed_event;
1698 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1699 1, &frame_destroyed_event, codec_width_, codec_height_));
1700 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001701 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001702 fake_encoder_.GetLastInputPixelFormat());
1703 video_stream_encoder_->Stop();
1704}
1705
Henrik Boström56db9ff2021-03-24 09:06:45 +01001706TEST_F(VideoStreamEncoderTest,
1707 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001709 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001710
1711 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1712
1713 rtc::Event frame_destroyed_event;
1714 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1715 1, &frame_destroyed_event, codec_width_, codec_height_));
1716 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001717 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001718 fake_encoder_.GetLastInputPixelFormat());
1719 video_stream_encoder_->Stop();
1720}
1721
Henrik Boström56db9ff2021-03-24 09:06:45 +01001722TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001725
1726 // Fake NV12 native frame does not allow mapping to I444.
1727 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1728
1729 rtc::Event frame_destroyed_event;
1730 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1731 1, &frame_destroyed_event, codec_width_, codec_height_));
1732 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001733 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001734 fake_encoder_.GetLastInputPixelFormat());
1735 video_stream_encoder_->Stop();
1736}
1737
Henrik Boström56db9ff2021-03-24 09:06:45 +01001738TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001740 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001741
1742 rtc::Event frame_destroyed_event;
1743 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1744 1, &frame_destroyed_event, codec_width_, codec_height_));
1745 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001746 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001747 fake_encoder_.GetLastInputPixelFormat());
1748 video_stream_encoder_->Stop();
1749}
1750
Ying Wang9b881ab2020-02-07 14:29:32 +01001751TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001753 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001754 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1755 WaitForEncodedFrame(1);
1756
Henrik Boström381d1092020-05-12 18:49:07 +02001757 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001758 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001759 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1760 // frames. Adding two frames means that the first frame will be dropped and
1761 // the second frame will be sent to the encoder.
1762 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1763 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1764 WaitForEncodedFrame(3);
1765 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1766 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1767 WaitForEncodedFrame(5);
1768 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1769 video_stream_encoder_->Stop();
1770}
1771
mflodmancc3d4422017-08-03 08:27:51 -07001772TEST_F(VideoStreamEncoderTest,
1773 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001775 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001776 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001777
1778 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001779 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001780 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001781 // The encoder will have been configured once when the first frame is
1782 // received.
1783 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001784
1785 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001786 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001787 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001788 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001789 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001790
1791 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001792 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001793 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001794 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001795 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001796
mflodmancc3d4422017-08-03 08:27:51 -07001797 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001798}
1799
mflodmancc3d4422017-08-03 08:27:51 -07001800TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001802 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001803
1804 // Capture a frame and wait for it to synchronize with the encoder thread.
1805 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001806 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001807 // The encoder will have been configured once.
1808 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001809 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1810 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001811
1812 codec_width_ *= 2;
1813 codec_height_ *= 2;
1814 // Capture a frame with a higher resolution and wait for it to synchronize
1815 // with the encoder thread.
1816 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001817 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001818 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1819 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001820 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001821
mflodmancc3d4422017-08-03 08:27:51 -07001822 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001823}
1824
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001825TEST_F(VideoStreamEncoderTest,
1826 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001828 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001829
1830 // Capture a frame and wait for it to synchronize with the encoder thread.
1831 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1832 WaitForEncodedFrame(1);
1833
1834 VideoEncoderConfig video_encoder_config;
1835 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1836 // Changing the max payload data length recreates encoder.
1837 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1838 kMaxPayloadLength / 2);
1839
1840 // Capture a frame and wait for it to synchronize with the encoder thread.
1841 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1842 WaitForEncodedFrame(2);
1843 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1844
1845 video_stream_encoder_->Stop();
1846}
1847
Sergey Silkin5ee69672019-07-02 14:18:34 +02001848TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001849 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001850 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001851
1852 VideoEncoderConfig video_encoder_config;
1853 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001854 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1855 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001856 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1857 kMaxPayloadLength);
1858
1859 // Capture a frame and wait for it to synchronize with the encoder thread.
1860 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1861 WaitForEncodedFrame(1);
1862 // The encoder will have been configured once when the first frame is
1863 // received.
1864 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001865 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001866 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001867 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001868 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1869
Sergey Silkin6456e352019-07-08 17:56:40 +02001870 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1871 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001872 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1873 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001874 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1875 kMaxPayloadLength);
1876
1877 // Capture a frame and wait for it to synchronize with the encoder thread.
1878 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1879 WaitForEncodedFrame(2);
1880 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1881 // Bitrate limits have changed - rate allocator should be reconfigured,
1882 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001883 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001884 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001885 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001886 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001887 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001888
1889 video_stream_encoder_->Stop();
1890}
1891
Sergey Silkin6456e352019-07-08 17:56:40 +02001892TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001893 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001894 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001895 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001896
Sergey Silkincd02eba2020-01-20 14:48:40 +01001897 const uint32_t kMinEncBitrateKbps = 100;
1898 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001899 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001900 /*frame_size_pixels=*/codec_width_ * codec_height_,
1901 /*min_start_bitrate_bps=*/0,
1902 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1903 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001904 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1905
Sergey Silkincd02eba2020-01-20 14:48:40 +01001906 VideoEncoderConfig video_encoder_config;
1907 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1908 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1909 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1910 (kMinEncBitrateKbps + 1) * 1000;
1911 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1912 kMaxPayloadLength);
1913
1914 // When both encoder and app provide bitrate limits, the intersection of
1915 // provided sets should be used.
1916 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1917 WaitForEncodedFrame(1);
1918 EXPECT_EQ(kMaxEncBitrateKbps,
1919 bitrate_allocator_factory_.codec_config().maxBitrate);
1920 EXPECT_EQ(kMinEncBitrateKbps + 1,
1921 bitrate_allocator_factory_.codec_config().minBitrate);
1922
1923 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1924 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1925 (kMinEncBitrateKbps - 1) * 1000;
1926 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1927 kMaxPayloadLength);
1928 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001929 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001930 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001931 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001932 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001933 bitrate_allocator_factory_.codec_config().minBitrate);
1934
Sergey Silkincd02eba2020-01-20 14:48:40 +01001935 video_stream_encoder_->Stop();
1936}
1937
1938TEST_F(VideoStreamEncoderTest,
1939 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001940 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001941 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001942
1943 const uint32_t kMinAppBitrateKbps = 100;
1944 const uint32_t kMaxAppBitrateKbps = 200;
1945 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1946 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1947 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1948 /*frame_size_pixels=*/codec_width_ * codec_height_,
1949 /*min_start_bitrate_bps=*/0,
1950 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1951 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1952 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1953
1954 VideoEncoderConfig video_encoder_config;
1955 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1956 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1957 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1958 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001959 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1960 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001961
Sergey Silkincd02eba2020-01-20 14:48:40 +01001962 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1963 WaitForEncodedFrame(1);
1964 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001965 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001966 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001967 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001968
1969 video_stream_encoder_->Stop();
1970}
1971
1972TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001973 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001975 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001976
1977 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001978 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001979 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001980 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001981 fake_encoder_.SetResolutionBitrateLimits(
1982 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1983
1984 VideoEncoderConfig video_encoder_config;
1985 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1986 video_encoder_config.max_bitrate_bps = 0;
1987 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1988 kMaxPayloadLength);
1989
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001990 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001991 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1992 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001993 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1994 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001995 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1997
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001998 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001999 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2000 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002001 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2002 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002003 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2004 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2005
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002006 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002007 // encoder for 360p should be used.
2008 video_source_.IncomingCapturedFrame(
2009 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2010 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002011 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2012 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002013 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2014 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2015
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002016 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002017 // ignored.
2018 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2019 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002020 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2021 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002022 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2023 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002024 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2025 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002026 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2027 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2028
2029 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2030 // for 270p should be used.
2031 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2032 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002033 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2034 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002035 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2036 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2037
2038 video_stream_encoder_->Stop();
2039}
2040
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002041TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002043 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002044
2045 VideoEncoderConfig video_encoder_config;
2046 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2047 video_encoder_config.max_bitrate_bps = 0;
2048 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2049 kMaxPayloadLength);
2050
2051 // Encode 720p frame to get the default encoder target bitrate.
2052 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2053 WaitForEncodedFrame(1);
2054 const uint32_t kDefaultTargetBitrateFor720pKbps =
2055 bitrate_allocator_factory_.codec_config()
2056 .simulcastStream[0]
2057 .targetBitrate;
2058
2059 // Set the max recommended encoder bitrate to something lower than the default
2060 // target bitrate.
2061 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2062 1280 * 720, 10 * 1000, 10 * 1000,
2063 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2064 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2065
2066 // Change resolution to trigger encoder reinitialization.
2067 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2068 WaitForEncodedFrame(2);
2069 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2070 WaitForEncodedFrame(3);
2071
2072 // Ensure the target bitrate is capped by the max bitrate.
2073 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2074 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2075 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2076 .simulcastStream[0]
2077 .targetBitrate *
2078 1000,
2079 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2080
2081 video_stream_encoder_->Stop();
2082}
2083
Åsa Perssona7e34d32021-01-20 15:36:13 +01002084TEST_F(VideoStreamEncoderTest,
2085 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2086 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2087 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2088 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2089 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2090 fake_encoder_.SetResolutionBitrateLimits(
2091 {kEncoderLimits270p, kEncoderLimits360p});
2092
2093 // Two streams, highest stream active.
2094 VideoEncoderConfig config;
2095 const int kNumStreams = 2;
2096 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2097 config.max_bitrate_bps = 0;
2098 config.simulcast_layers[0].active = false;
2099 config.simulcast_layers[1].active = true;
2100 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002101 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002102 "VP8", /*max qp*/ 56, /*screencast*/ false,
2103 /*screenshare enabled*/ false);
2104 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2105
2106 // The encoder bitrate limits for 270p should be used.
2107 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2108 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002109 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002110 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002111 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002112 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002113 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002114
2115 // The encoder bitrate limits for 360p should be used.
2116 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2117 EXPECT_FALSE(WaitForFrame(1000));
2118 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002119 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002120 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002121 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002122
2123 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2124 video_source_.IncomingCapturedFrame(
2125 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2126 EXPECT_FALSE(WaitForFrame(1000));
2127 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002128 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002129 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002130 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002131
2132 // Resolution higher than 360p. Encoder limits should be ignored.
2133 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2134 EXPECT_FALSE(WaitForFrame(1000));
2135 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002136 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002137 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002138 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002139 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002140 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002141 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002142 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002143
2144 // Resolution lower than 270p. The encoder limits for 270p should be used.
2145 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2146 EXPECT_FALSE(WaitForFrame(1000));
2147 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002148 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002149 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002150 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002151
2152 video_stream_encoder_->Stop();
2153}
2154
2155TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002156 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2157 // Two streams, highest stream active.
2158 VideoEncoderConfig config;
2159 const int kNumStreams = 2;
2160 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2161 config.max_bitrate_bps = 0;
2162 config.simulcast_layers[0].active = false;
2163 config.simulcast_layers[1].active = true;
2164 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002165 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002166 "VP8", /*max qp*/ 56, /*screencast*/ false,
2167 /*screenshare enabled*/ false);
2168 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2169
2170 // Default bitrate limits for 270p should be used.
2171 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2172 kDefaultLimits270p =
2173 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002174 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002175 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2176 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002177 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002178 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002179 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002180 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002181 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002182
2183 // Default bitrate limits for 360p should be used.
2184 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2185 kDefaultLimits360p =
2186 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002187 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002188 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002191 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002192 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002193 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002194
2195 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2196 video_source_.IncomingCapturedFrame(
2197 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2198 EXPECT_FALSE(WaitForFrame(1000));
2199 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002200 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002201 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002202 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002203
2204 // Default bitrate limits for 540p should be used.
2205 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2206 kDefaultLimits540p =
2207 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002208 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002209 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2210 EXPECT_FALSE(WaitForFrame(1000));
2211 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002212 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002213 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002214 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002215
2216 video_stream_encoder_->Stop();
2217}
2218
2219TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002220 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2221 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2222 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2223 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2224 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2225 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2226 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2227 fake_encoder_.SetResolutionBitrateLimits(
2228 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2229
2230 // Three streams, middle stream active.
2231 VideoEncoderConfig config;
2232 const int kNumStreams = 3;
2233 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2234 config.simulcast_layers[0].active = false;
2235 config.simulcast_layers[1].active = true;
2236 config.simulcast_layers[2].active = false;
2237 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002238 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002239 "VP8", /*max qp*/ 56, /*screencast*/ false,
2240 /*screenshare enabled*/ false);
2241 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2242
2243 // The encoder bitrate limits for 360p should be used.
2244 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2245 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002246 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002247 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002248 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002249 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002250 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002251
2252 // The encoder bitrate limits for 270p should be used.
2253 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2254 EXPECT_FALSE(WaitForFrame(1000));
2255 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002256 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002257 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002258 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002259
2260 video_stream_encoder_->Stop();
2261}
2262
2263TEST_F(VideoStreamEncoderTest,
2264 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2265 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2266 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2267 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2268 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2269 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2270 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2271 fake_encoder_.SetResolutionBitrateLimits(
2272 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2273
2274 // Three streams, lowest stream active.
2275 VideoEncoderConfig config;
2276 const int kNumStreams = 3;
2277 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2278 config.simulcast_layers[0].active = true;
2279 config.simulcast_layers[1].active = false;
2280 config.simulcast_layers[2].active = false;
2281 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002282 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002283 "VP8", /*max qp*/ 56, /*screencast*/ false,
2284 /*screenshare enabled*/ false);
2285 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2286
2287 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2288 // on lowest stream, limits for 270p should not be used
2289 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2290 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002291 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002292 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002293 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002294 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002295 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002296
2297 video_stream_encoder_->Stop();
2298}
2299
2300TEST_F(VideoStreamEncoderTest,
2301 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2302 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2303 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2304 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2305 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2306 fake_encoder_.SetResolutionBitrateLimits(
2307 {kEncoderLimits270p, kEncoderLimits360p});
2308 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2309
2310 // Two streams, highest stream active.
2311 VideoEncoderConfig config;
2312 const int kNumStreams = 2;
2313 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2314 config.simulcast_layers[0].active = false;
2315 config.simulcast_layers[1].active = true;
2316 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2317 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002318 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002319 "VP8", /*max qp*/ 56, /*screencast*/ false,
2320 /*screenshare enabled*/ false);
2321 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2322
2323 // The encoder bitrate limits for 270p should be used.
2324 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2325 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002326 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002327 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002328 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002329 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002330 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002331
2332 // The max configured bitrate is less than the encoder limit for 360p.
2333 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2334 EXPECT_FALSE(WaitForFrame(1000));
2335 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002336 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002337 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002338 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002339
2340 video_stream_encoder_->Stop();
2341}
2342
mflodmancc3d4422017-08-03 08:27:51 -07002343TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002344 EXPECT_TRUE(video_source_.has_sinks());
2345 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002346 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002347 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002348 EXPECT_FALSE(video_source_.has_sinks());
2349 EXPECT_TRUE(new_video_source.has_sinks());
2350
mflodmancc3d4422017-08-03 08:27:51 -07002351 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002352}
2353
mflodmancc3d4422017-08-03 08:27:51 -07002354TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002355 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002356 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002357 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002358 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002359}
2360
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002361class ResolutionAlignmentTest
2362 : public VideoStreamEncoderTest,
2363 public ::testing::WithParamInterface<
2364 ::testing::tuple<int, std::vector<double>>> {
2365 public:
2366 ResolutionAlignmentTest()
2367 : requested_alignment_(::testing::get<0>(GetParam())),
2368 scale_factors_(::testing::get<1>(GetParam())) {}
2369
2370 protected:
2371 const int requested_alignment_;
2372 const std::vector<double> scale_factors_;
2373};
2374
2375INSTANTIATE_TEST_SUITE_P(
2376 AlignmentAndScaleFactors,
2377 ResolutionAlignmentTest,
2378 ::testing::Combine(
2379 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2380 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2381 std::vector<double>{-1.0, -1.0},
2382 std::vector<double>{-1.0, -1.0, -1.0},
2383 std::vector<double>{4.0, 2.0, 1.0},
2384 std::vector<double>{9999.0, -1.0, 1.0},
2385 std::vector<double>{3.99, 2.01, 1.0},
2386 std::vector<double>{4.9, 1.7, 1.25},
2387 std::vector<double>{10.0, 4.0, 3.0},
2388 std::vector<double>{1.75, 3.5},
2389 std::vector<double>{1.5, 2.5},
2390 std::vector<double>{1.3, 1.0})));
2391
2392TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2393 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002394 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002395 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2396 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2397
2398 // Fill config with the scaling factor by which to reduce encoding size.
2399 const int num_streams = scale_factors_.size();
2400 VideoEncoderConfig config;
2401 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2402 for (int i = 0; i < num_streams; ++i) {
2403 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2404 }
2405 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002406 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002407 "VP8", /*max qp*/ 56, /*screencast*/ false,
2408 /*screenshare enabled*/ false);
2409 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2410
Henrik Boström381d1092020-05-12 18:49:07 +02002411 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002412 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2413 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002414 // Wait for all layers before triggering event.
2415 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002416
2417 // On the 1st frame, we should have initialized the encoder and
2418 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002419 int64_t timestamp_ms = kFrameIntervalMs;
2420 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2421 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002422 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002423
2424 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2425 // (It's up the to the encoder to potentially drop the previous frame,
2426 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002427 timestamp_ms += kFrameIntervalMs;
2428 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2429 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002430 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002431
Asa Persson606d3cb2021-10-04 10:07:11 +02002432 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002433 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2434 // Frame size should be a multiple of the requested alignment.
2435 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2436 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2437 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2438 // Aspect ratio should match.
2439 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2440 codec.height * codec.simulcastStream[i].width);
2441 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002442
2443 video_stream_encoder_->Stop();
2444}
2445
Jonathan Yubc771b72017-12-08 17:04:29 -08002446TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2447 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002448 const int kWidth = 1280;
2449 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002450
2451 // We rely on the automatic resolution adaptation, but we handle framerate
2452 // adaptation manually by mocking the stats proxy.
2453 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002454
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002455 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002457 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002458 video_stream_encoder_->SetSource(&video_source_,
2459 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002460 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002461 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002462 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002463 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2464
Jonathan Yubc771b72017-12-08 17:04:29 -08002465 // Adapt down as far as possible.
2466 rtc::VideoSinkWants last_wants;
2467 int64_t t = 1;
2468 int loop_count = 0;
2469 do {
2470 ++loop_count;
2471 last_wants = video_source_.sink_wants();
2472
2473 // Simulate the framerate we've been asked to adapt to.
2474 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2475 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2476 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2477 mock_stats.input_frame_rate = fps;
2478 stats_proxy_->SetMockStats(mock_stats);
2479
2480 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2481 sink_.WaitForEncodedFrame(t);
2482 t += frame_interval_ms;
2483
mflodmancc3d4422017-08-03 08:27:51 -07002484 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002485 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002486 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002487 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2488 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002489 } while (video_source_.sink_wants().max_pixel_count <
2490 last_wants.max_pixel_count ||
2491 video_source_.sink_wants().max_framerate_fps <
2492 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002493
Jonathan Yubc771b72017-12-08 17:04:29 -08002494 // Verify that we've adapted all the way down.
2495 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002496 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002497 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2498 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002499 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002500 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2501 *video_source_.last_sent_height());
2502 EXPECT_EQ(kMinBalancedFramerateFps,
2503 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002504
Jonathan Yubc771b72017-12-08 17:04:29 -08002505 // Adapt back up the same number of times we adapted down.
2506 for (int i = 0; i < loop_count - 1; ++i) {
2507 last_wants = video_source_.sink_wants();
2508
2509 // Simulate the framerate we've been asked to adapt to.
2510 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2511 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2512 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2513 mock_stats.input_frame_rate = fps;
2514 stats_proxy_->SetMockStats(mock_stats);
2515
2516 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2517 sink_.WaitForEncodedFrame(t);
2518 t += frame_interval_ms;
2519
Henrik Boström91aa7322020-04-28 12:24:33 +02002520 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002521 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002522 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002523 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2524 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002525 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2526 last_wants.max_pixel_count ||
2527 video_source_.sink_wants().max_framerate_fps >
2528 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002529 }
2530
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002531 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002532 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002533 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002534 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2535 EXPECT_EQ((loop_count - 1) * 2,
2536 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002537
mflodmancc3d4422017-08-03 08:27:51 -07002538 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002539}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002540
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002541TEST_F(VideoStreamEncoderTest,
2542 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002543 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2544 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002545 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002546
2547 const int kFrameWidth = 1280;
2548 const int kFrameHeight = 720;
2549
2550 int64_t ntp_time = kFrameIntervalMs;
2551
2552 // Force an input frame rate to be available, or the adaptation call won't
2553 // know what framerate to adapt form.
2554 const int kInputFps = 30;
2555 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2556 stats.input_frame_rate = kInputFps;
2557 stats_proxy_->SetMockStats(stats);
2558
2559 video_source_.set_adaptation_enabled(true);
2560 video_stream_encoder_->SetSource(
2561 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002562 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002563 video_source_.IncomingCapturedFrame(
2564 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2565 sink_.WaitForEncodedFrame(ntp_time);
2566 ntp_time += kFrameIntervalMs;
2567
2568 // Trigger CPU overuse.
2569 video_stream_encoder_->TriggerCpuOveruse();
2570 video_source_.IncomingCapturedFrame(
2571 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2572 sink_.WaitForEncodedFrame(ntp_time);
2573 ntp_time += kFrameIntervalMs;
2574
2575 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2576 EXPECT_EQ(std::numeric_limits<int>::max(),
2577 video_source_.sink_wants().max_pixel_count);
2578 // Some framerate constraint should be set.
2579 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2580 EXPECT_LT(restricted_fps, kInputFps);
2581 video_source_.IncomingCapturedFrame(
2582 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2583 sink_.WaitForEncodedFrame(ntp_time);
2584 ntp_time += 100;
2585
Henrik Boström2671dac2020-05-19 16:29:09 +02002586 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002587 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2588 // Give the encoder queue time to process the change in degradation preference
2589 // by waiting for an encoded frame.
2590 video_source_.IncomingCapturedFrame(
2591 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2592 sink_.WaitForEncodedFrame(ntp_time);
2593 ntp_time += kFrameIntervalMs;
2594
2595 video_stream_encoder_->TriggerQualityLow();
2596 video_source_.IncomingCapturedFrame(
2597 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2598 sink_.WaitForEncodedFrame(ntp_time);
2599 ntp_time += kFrameIntervalMs;
2600
2601 // Some resolution constraint should be set.
2602 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2603 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2604 kFrameWidth * kFrameHeight);
2605 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2606
2607 int pixel_count = video_source_.sink_wants().max_pixel_count;
2608 // Triggering a CPU underuse should not change the sink wants since it has
2609 // not been overused for resolution since we changed degradation preference.
2610 video_stream_encoder_->TriggerCpuUnderuse();
2611 video_source_.IncomingCapturedFrame(
2612 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2613 sink_.WaitForEncodedFrame(ntp_time);
2614 ntp_time += kFrameIntervalMs;
2615 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2616 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2617
Evan Shrubsole64469032020-06-11 10:45:29 +02002618 // Change the degradation preference back. CPU underuse should not adapt since
2619 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002620 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002621 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2622 video_source_.IncomingCapturedFrame(
2623 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2624 sink_.WaitForEncodedFrame(ntp_time);
2625 ntp_time += 100;
2626 // Resolution adaptations is gone after changing degradation preference.
2627 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2628 EXPECT_EQ(std::numeric_limits<int>::max(),
2629 video_source_.sink_wants().max_pixel_count);
2630 // The fps adaptation from above is now back.
2631 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2632
2633 // Trigger CPU underuse.
2634 video_stream_encoder_->TriggerCpuUnderuse();
2635 video_source_.IncomingCapturedFrame(
2636 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2637 sink_.WaitForEncodedFrame(ntp_time);
2638 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002639 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2640
2641 // Trigger QP underuse, fps should return to normal.
2642 video_stream_encoder_->TriggerQualityHigh();
2643 video_source_.IncomingCapturedFrame(
2644 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2645 sink_.WaitForEncodedFrame(ntp_time);
2646 ntp_time += kFrameIntervalMs;
2647 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002648
2649 video_stream_encoder_->Stop();
2650}
2651
mflodmancc3d4422017-08-03 08:27:51 -07002652TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002654 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002655 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002656
sprangc5d62e22017-04-02 23:53:04 -07002657 const int kFrameWidth = 1280;
2658 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002659
Åsa Persson8c1bf952018-09-13 10:42:19 +02002660 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002661
kthelgason5e13d412016-12-01 03:59:51 -08002662 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002663 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002664 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002665 frame_timestamp += kFrameIntervalMs;
2666
perkj803d97f2016-11-01 11:45:46 -07002667 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002668 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002669 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002670 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002671 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002672 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002673
asapersson0944a802017-04-07 00:57:58 -07002674 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002675 // wanted resolution.
2676 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2677 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2678 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002679 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002680
2681 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002682 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002683 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002684 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002685 // Give the encoder queue time to process the change in degradation preference
2686 // by waiting for an encoded frame.
2687 new_video_source.IncomingCapturedFrame(
2688 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2689 sink_.WaitForEncodedFrame(frame_timestamp);
2690 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002691 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002692 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002693
sprangc5d62e22017-04-02 23:53:04 -07002694 // Force an input frame rate to be available, or the adaptation call won't
2695 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002696 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002697 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002698 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002699 stats_proxy_->SetMockStats(stats);
2700
mflodmancc3d4422017-08-03 08:27:51 -07002701 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002702 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002703 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002704 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002705 frame_timestamp += kFrameIntervalMs;
2706
2707 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002708 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002709 EXPECT_EQ(std::numeric_limits<int>::max(),
2710 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002711 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002712
asapersson02465b82017-04-10 01:12:52 -07002713 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002714 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2715 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002716 // Give the encoder queue time to process the change in degradation preference
2717 // by waiting for an encoded frame.
2718 new_video_source.IncomingCapturedFrame(
2719 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2720 sink_.WaitForEncodedFrame(frame_timestamp);
2721 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002722 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002723
mflodmancc3d4422017-08-03 08:27:51 -07002724 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002725 new_video_source.IncomingCapturedFrame(
2726 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002727 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002728 frame_timestamp += kFrameIntervalMs;
2729
2730 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002731 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002732
2733 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002734 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002735 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002736 // Give the encoder queue time to process the change in degradation preference
2737 // by waiting for an encoded frame.
2738 new_video_source.IncomingCapturedFrame(
2739 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2740 sink_.WaitForEncodedFrame(frame_timestamp);
2741 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002742 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2743 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002744 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002745 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002746
2747 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002748 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002749 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002750 // Give the encoder queue time to process the change in degradation preference
2751 // by waiting for an encoded frame.
2752 new_video_source.IncomingCapturedFrame(
2753 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2754 sink_.WaitForEncodedFrame(frame_timestamp);
2755 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002756 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2757 EXPECT_EQ(std::numeric_limits<int>::max(),
2758 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002759 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002760
mflodmancc3d4422017-08-03 08:27:51 -07002761 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002762}
2763
mflodmancc3d4422017-08-03 08:27:51 -07002764TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002766 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002767
asaperssonfab67072017-04-04 05:51:49 -07002768 const int kWidth = 1280;
2769 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002770 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002771 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002772 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2773 EXPECT_FALSE(stats.bw_limited_resolution);
2774 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2775
2776 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002777 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002778 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002779 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002780
2781 stats = stats_proxy_->GetStats();
2782 EXPECT_TRUE(stats.bw_limited_resolution);
2783 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2784
2785 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002786 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002787 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002788 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002789
2790 stats = stats_proxy_->GetStats();
2791 EXPECT_FALSE(stats.bw_limited_resolution);
2792 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2793 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2794
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002796}
2797
mflodmancc3d4422017-08-03 08:27:51 -07002798TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002799 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002800 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002801
2802 const int kWidth = 1280;
2803 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002804 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002805 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002806 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2807 EXPECT_FALSE(stats.cpu_limited_resolution);
2808 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2809
2810 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002811 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002812 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002814
2815 stats = stats_proxy_->GetStats();
2816 EXPECT_TRUE(stats.cpu_limited_resolution);
2817 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2818
2819 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002820 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002821 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002822 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002823
2824 stats = stats_proxy_->GetStats();
2825 EXPECT_FALSE(stats.cpu_limited_resolution);
2826 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002827 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002828
mflodmancc3d4422017-08-03 08:27:51 -07002829 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002830}
2831
mflodmancc3d4422017-08-03 08:27:51 -07002832TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002835
asaperssonfab67072017-04-04 05:51:49 -07002836 const int kWidth = 1280;
2837 const int kHeight = 720;
2838 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002839 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002840 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002841 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002842 EXPECT_FALSE(stats.cpu_limited_resolution);
2843 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2844
asaperssonfab67072017-04-04 05:51:49 -07002845 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002846 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002847 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002848 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002849 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002850 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002851 EXPECT_TRUE(stats.cpu_limited_resolution);
2852 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2853
2854 // Set new source with adaptation still enabled.
2855 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002856 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002857 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002858
asaperssonfab67072017-04-04 05:51:49 -07002859 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002860 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002861 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002862 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002863 EXPECT_TRUE(stats.cpu_limited_resolution);
2864 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2865
2866 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002867 video_stream_encoder_->SetSource(&new_video_source,
2868 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002869
asaperssonfab67072017-04-04 05:51:49 -07002870 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002871 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002872 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002873 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002874 EXPECT_FALSE(stats.cpu_limited_resolution);
2875 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2876
2877 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002878 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002879 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002880
asaperssonfab67072017-04-04 05:51:49 -07002881 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002883 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002884 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002885 EXPECT_TRUE(stats.cpu_limited_resolution);
2886 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2887
asaperssonfab67072017-04-04 05:51:49 -07002888 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002889 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002890 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002891 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002892 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002893 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002894 EXPECT_FALSE(stats.cpu_limited_resolution);
2895 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002896 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002897
mflodmancc3d4422017-08-03 08:27:51 -07002898 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002899}
2900
mflodmancc3d4422017-08-03 08:27:51 -07002901TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002902 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002903 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002904
asaperssonfab67072017-04-04 05:51:49 -07002905 const int kWidth = 1280;
2906 const int kHeight = 720;
2907 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002908 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002909 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002910 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002911 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002912 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002913
2914 // Set new source with adaptation still enabled.
2915 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002916 video_stream_encoder_->SetSource(&new_video_source,
2917 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002918
asaperssonfab67072017-04-04 05:51:49 -07002919 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002920 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002921 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002922 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002923 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002924 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002925
asaperssonfab67072017-04-04 05:51:49 -07002926 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002927 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002928 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002929 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002930 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002931 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002932 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002933 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002934
asaperssonfab67072017-04-04 05:51:49 -07002935 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002936 video_stream_encoder_->SetSource(&new_video_source,
2937 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002938
asaperssonfab67072017-04-04 05:51:49 -07002939 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002940 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002941 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002942 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002943 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002944 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002945
asapersson02465b82017-04-10 01:12:52 -07002946 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002947 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002948 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002949
asaperssonfab67072017-04-04 05:51:49 -07002950 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002951 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002952 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002953 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002954 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002955 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2956 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002957
mflodmancc3d4422017-08-03 08:27:51 -07002958 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002959}
2960
mflodmancc3d4422017-08-03 08:27:51 -07002961TEST_F(VideoStreamEncoderTest,
2962 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002964 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002965
2966 const int kWidth = 1280;
2967 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002968 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002969 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002970 video_source_.IncomingCapturedFrame(
2971 CreateFrame(timestamp_ms, kWidth, kHeight));
2972 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002973 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2974 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2975 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2976
2977 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002978 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002979 timestamp_ms += kFrameIntervalMs;
2980 video_source_.IncomingCapturedFrame(
2981 CreateFrame(timestamp_ms, kWidth, kHeight));
2982 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002983 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2984 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2985 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2986
2987 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002988 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002989 timestamp_ms += kFrameIntervalMs;
2990 video_source_.IncomingCapturedFrame(
2991 CreateFrame(timestamp_ms, kWidth, kHeight));
2992 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002993 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2994 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2995 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2996
Niels Möller4db138e2018-04-19 09:04:13 +02002997 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002998 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002999
3000 VideoEncoderConfig video_encoder_config;
3001 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3002 // Make format different, to force recreation of encoder.
3003 video_encoder_config.video_format.parameters["foo"] = "foo";
3004 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003005 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003006 timestamp_ms += kFrameIntervalMs;
3007 video_source_.IncomingCapturedFrame(
3008 CreateFrame(timestamp_ms, kWidth, kHeight));
3009 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003010 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3011 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3012 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3013
mflodmancc3d4422017-08-03 08:27:51 -07003014 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003015}
3016
mflodmancc3d4422017-08-03 08:27:51 -07003017TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003018 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003020 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003021
3022 const int kWidth = 1280;
3023 const int kHeight = 720;
3024 int sequence = 1;
3025
3026 // Enable BALANCED preference, no initial limitation.
3027 test::FrameForwarder source;
3028 video_stream_encoder_->SetSource(&source,
3029 webrtc::DegradationPreference::BALANCED);
3030 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3031 WaitForEncodedFrame(sequence++);
3032 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3033 EXPECT_FALSE(stats.cpu_limited_resolution);
3034 EXPECT_FALSE(stats.cpu_limited_framerate);
3035 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3036
3037 // Trigger CPU overuse, should now adapt down.
3038 video_stream_encoder_->TriggerCpuOveruse();
3039 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3040 WaitForEncodedFrame(sequence++);
3041 stats = stats_proxy_->GetStats();
3042 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3043
3044 // Set new degradation preference should clear restrictions since we changed
3045 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003046 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003047 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3048 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3049 WaitForEncodedFrame(sequence++);
3050 stats = stats_proxy_->GetStats();
3051 EXPECT_FALSE(stats.cpu_limited_resolution);
3052 EXPECT_FALSE(stats.cpu_limited_framerate);
3053 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3054
3055 // Force an input frame rate to be available, or the adaptation call won't
3056 // know what framerate to adapt from.
3057 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3058 mock_stats.input_frame_rate = 30;
3059 stats_proxy_->SetMockStats(mock_stats);
3060 video_stream_encoder_->TriggerCpuOveruse();
3061 stats_proxy_->ResetMockStats();
3062 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3063 WaitForEncodedFrame(sequence++);
3064
3065 // We have now adapted once.
3066 stats = stats_proxy_->GetStats();
3067 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3068
3069 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003070 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3071 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003072 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3073 WaitForEncodedFrame(sequence++);
3074 stats = stats_proxy_->GetStats();
3075 EXPECT_FALSE(stats.cpu_limited_resolution);
3076 EXPECT_FALSE(stats.cpu_limited_framerate);
3077 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3078
3079 video_stream_encoder_->Stop();
3080}
3081
3082TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003083 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003084 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003085 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003086
asapersson0944a802017-04-07 00:57:58 -07003087 const int kWidth = 1280;
3088 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003089 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003090
asaperssonfab67072017-04-04 05:51:49 -07003091 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003092 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003093 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003094 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003095 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003096 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3097
asapersson02465b82017-04-10 01:12:52 -07003098 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003100 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003101 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003102 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003103 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003104 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003105 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3106
3107 // Set new source with adaptation still enabled.
3108 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003109 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003110 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003111
3112 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003113 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003114 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003115 stats = stats_proxy_->GetStats();
3116 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003117 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003118 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3119
sprangc5d62e22017-04-02 23:53:04 -07003120 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003121 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003122 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003123 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003124 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003125 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003126 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003127 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003128 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003129 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003130 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3131
sprangc5d62e22017-04-02 23:53:04 -07003132 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003133 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003134 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3135 mock_stats.input_frame_rate = 30;
3136 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003137 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003138 stats_proxy_->ResetMockStats();
3139
3140 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003141 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003142 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003143
3144 // Framerate now adapted.
3145 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003146 EXPECT_FALSE(stats.cpu_limited_resolution);
3147 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003148 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3149
3150 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003151 video_stream_encoder_->SetSource(&new_video_source,
3152 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003153 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003154 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003156
3157 stats = stats_proxy_->GetStats();
3158 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003159 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003160 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3161
3162 // Try to trigger overuse. Should not succeed.
3163 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003165 stats_proxy_->ResetMockStats();
3166
3167 stats = stats_proxy_->GetStats();
3168 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003169 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003170 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3171
3172 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003173 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003174 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003175 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003176 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003177 stats = stats_proxy_->GetStats();
3178 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003179 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003180 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003181
3182 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003183 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003184 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003185 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003186 stats = stats_proxy_->GetStats();
3187 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003188 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003189 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3190
3191 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003192 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003193 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003194 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003195 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003196 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003197 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003198 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003199 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003200 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003201 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3202
3203 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003204 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003205 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003206 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003207 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003208 stats = stats_proxy_->GetStats();
3209 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003210 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003211 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003212 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003213
mflodmancc3d4422017-08-03 08:27:51 -07003214 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003215}
3216
mflodmancc3d4422017-08-03 08:27:51 -07003217TEST_F(VideoStreamEncoderTest,
3218 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003219 const int kWidth = 1280;
3220 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003222 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003223
asaperssonfab67072017-04-04 05:51:49 -07003224 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003225 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003226
asaperssonfab67072017-04-04 05:51:49 -07003227 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003228 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003229
asaperssonfab67072017-04-04 05:51:49 -07003230 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003231 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003232
asaperssonfab67072017-04-04 05:51:49 -07003233 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003234 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003235
kthelgason876222f2016-11-29 01:44:11 -08003236 // Expect a scale down.
3237 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003238 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003239
asapersson02465b82017-04-10 01:12:52 -07003240 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003241 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003242 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003243 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003244
asaperssonfab67072017-04-04 05:51:49 -07003245 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003246 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003247 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003249
asaperssonfab67072017-04-04 05:51:49 -07003250 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003251 EXPECT_EQ(std::numeric_limits<int>::max(),
3252 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003253
asaperssonfab67072017-04-04 05:51:49 -07003254 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003256 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003257 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003258
asapersson02465b82017-04-10 01:12:52 -07003259 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003260 EXPECT_EQ(std::numeric_limits<int>::max(),
3261 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003262
mflodmancc3d4422017-08-03 08:27:51 -07003263 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003264}
3265
mflodmancc3d4422017-08-03 08:27:51 -07003266TEST_F(VideoStreamEncoderTest,
3267 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003268 const int kWidth = 1280;
3269 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003270 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003271 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003272
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003273 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003274 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003275 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003276 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003277
3278 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003279 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003280 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003281 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3282 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3283
3284 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003285 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003286 EXPECT_THAT(source.sink_wants(),
3287 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003288 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3289 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3290 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3291
3292 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003293 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003294 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3295 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3296 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3297
mflodmancc3d4422017-08-03 08:27:51 -07003298 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003299}
3300
mflodmancc3d4422017-08-03 08:27:51 -07003301TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003302 const int kWidth = 1280;
3303 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003305 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003306
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003307 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003308 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003309 video_stream_encoder_->SetSource(&source,
3310 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003311 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3312 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003313 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003314
3315 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003316 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003317 EXPECT_THAT(source.sink_wants(),
3318 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003319 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3320 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3321 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3322
3323 // Trigger adapt down for same input resolution, expect no change.
3324 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3325 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003326 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003327 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3329 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3330
3331 // Trigger adapt down for larger input resolution, expect no change.
3332 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3333 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003334 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003335 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3336 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3337 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3338
mflodmancc3d4422017-08-03 08:27:51 -07003339 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003340}
3341
mflodmancc3d4422017-08-03 08:27:51 -07003342TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003343 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3344 const int kWidth = 640;
3345 const int kHeight = 360;
3346 const int64_t kFrameIntervalMs = 150;
3347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003349
3350 // Enable BALANCED preference, no initial limitation.
3351 AdaptingFrameForwarder source(&time_controller_);
3352 source.set_adaptation_enabled(true);
3353 video_stream_encoder_->SetSource(&source,
3354 webrtc::DegradationPreference::BALANCED);
3355
3356 int64_t timestamp_ms = kFrameIntervalMs;
3357 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3358 sink_.WaitForEncodedFrame(kWidth, kHeight);
3359 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3360 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3361 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3362 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
3364 // Trigger adapt down, expect reduced fps (640x360@15fps).
3365 video_stream_encoder_->TriggerQualityLow();
3366 timestamp_ms += kFrameIntervalMs;
3367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3368 sink_.WaitForEncodedFrame(timestamp_ms);
3369 EXPECT_THAT(source.sink_wants(),
3370 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3371 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3373 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3374
3375 // Source requests 270p, expect reduced resolution (480x270@15fps).
3376 source.OnOutputFormatRequest(480, 270);
3377 timestamp_ms += kFrameIntervalMs;
3378 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3379 WaitForEncodedFrame(480, 270);
3380 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3381
3382 // Trigger adapt down, expect reduced fps (480x270@10fps).
3383 video_stream_encoder_->TriggerQualityLow();
3384 timestamp_ms += kFrameIntervalMs;
3385 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3386 sink_.WaitForEncodedFrame(timestamp_ms);
3387 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3388 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3390 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3391
3392 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3393 source.OnOutputFormatRequest(320, 180);
3394 timestamp_ms += kFrameIntervalMs;
3395 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3396 WaitForEncodedFrame(320, 180);
3397 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3398
3399 // Trigger adapt down, expect reduced fps (320x180@7fps).
3400 video_stream_encoder_->TriggerQualityLow();
3401 timestamp_ms += kFrameIntervalMs;
3402 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3403 sink_.WaitForEncodedFrame(timestamp_ms);
3404 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3405 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3406 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3407 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3408
3409 // Source requests VGA, expect increased resolution (640x360@7fps).
3410 source.OnOutputFormatRequest(640, 360);
3411 timestamp_ms += kFrameIntervalMs;
3412 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3413 WaitForEncodedFrame(timestamp_ms);
3414 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3415
3416 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3417 video_stream_encoder_->TriggerQualityHigh();
3418 timestamp_ms += kFrameIntervalMs;
3419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3420 WaitForEncodedFrame(timestamp_ms);
3421 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3422 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3423 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3424 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3425
3426 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3427 video_stream_encoder_->TriggerQualityHigh();
3428 timestamp_ms += kFrameIntervalMs;
3429 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3430 WaitForEncodedFrame(timestamp_ms);
3431 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3432 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3434 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
3436 // Trigger adapt up, expect increased fps (640x360@maxfps).
3437 video_stream_encoder_->TriggerQualityHigh();
3438 timestamp_ms += kFrameIntervalMs;
3439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3440 WaitForEncodedFrame(timestamp_ms);
3441 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3442 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3444 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3445
3446 video_stream_encoder_->Stop();
3447}
3448
3449TEST_F(VideoStreamEncoderTest,
3450 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3451 const int kWidth = 1280;
3452 const int kHeight = 720;
3453 const int64_t kFrameIntervalMs = 150;
3454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003455 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003456
3457 // Enable BALANCED preference, no initial limitation.
3458 AdaptingFrameForwarder source(&time_controller_);
3459 source.set_adaptation_enabled(true);
3460 video_stream_encoder_->SetSource(&source,
3461 webrtc::DegradationPreference::BALANCED);
3462
3463 int64_t timestamp_ms = kFrameIntervalMs;
3464 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3465 sink_.WaitForEncodedFrame(kWidth, kHeight);
3466 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3468 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3469 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3470
3471 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3472 video_stream_encoder_->TriggerQualityLow();
3473 timestamp_ms += kFrameIntervalMs;
3474 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3475 sink_.WaitForEncodedFrame(timestamp_ms);
3476 EXPECT_THAT(source.sink_wants(),
3477 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3480 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3481
3482 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3483 video_stream_encoder_->TriggerQualityLow();
3484 timestamp_ms += kFrameIntervalMs;
3485 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3486 sink_.WaitForEncodedFrame(timestamp_ms);
3487 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3489 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3490 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3491
3492 // Trigger adapt down, expect reduced fps (640x360@15fps).
3493 video_stream_encoder_->TriggerQualityLow();
3494 timestamp_ms += kFrameIntervalMs;
3495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3496 WaitForEncodedFrame(timestamp_ms);
3497 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3499 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3500 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3501
3502 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3503 source.OnOutputFormatRequest(320, 180);
3504 timestamp_ms += kFrameIntervalMs;
3505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3506 WaitForEncodedFrame(320, 180);
3507 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3508 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3509
3510 // Trigger adapt down, expect reduced fps (320x180@7fps).
3511 video_stream_encoder_->TriggerCpuOveruse();
3512 timestamp_ms += kFrameIntervalMs;
3513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3514 WaitForEncodedFrame(timestamp_ms);
3515 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3518 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3519 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3520 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3521 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3522
3523 // Source requests HD, expect increased resolution (640x360@7fps).
3524 source.OnOutputFormatRequest(1280, 720);
3525 timestamp_ms += kFrameIntervalMs;
3526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3527 WaitForEncodedFrame(timestamp_ms);
3528 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3529 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3530
3531 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3532 video_stream_encoder_->TriggerCpuUnderuse();
3533 timestamp_ms += kFrameIntervalMs;
3534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3535 WaitForEncodedFrame(timestamp_ms);
3536 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3538 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3539 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3540 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3541 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3542 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3543
3544 // Trigger adapt up, expect increased fps (640x360@maxfps).
3545 video_stream_encoder_->TriggerQualityHigh();
3546 video_stream_encoder_->TriggerCpuUnderuse();
3547 timestamp_ms += kFrameIntervalMs;
3548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3549 WaitForEncodedFrame(timestamp_ms);
3550 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3552 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3553 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3554 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3555 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3556 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3557
3558 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3559 video_stream_encoder_->TriggerQualityHigh();
3560 video_stream_encoder_->TriggerCpuUnderuse();
3561 timestamp_ms += kFrameIntervalMs;
3562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3563 WaitForEncodedFrame(timestamp_ms);
3564 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3567 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3568 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3569 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3570 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3571
3572 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3573 video_stream_encoder_->TriggerQualityHigh();
3574 video_stream_encoder_->TriggerCpuUnderuse();
3575 timestamp_ms += kFrameIntervalMs;
3576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3577 WaitForEncodedFrame(timestamp_ms);
3578 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3579 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3581 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3582 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3583 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3584 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3585
3586 video_stream_encoder_->Stop();
3587}
3588
3589TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003590 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003591 const int kWidth = 1280;
3592 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003594 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003595
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003596 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003597 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003598 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003599 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003600
3601 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003602 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003603 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003604 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3605 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3606
3607 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003608 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003609 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003610 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3611 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3612
mflodmancc3d4422017-08-03 08:27:51 -07003613 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003614}
3615
mflodmancc3d4422017-08-03 08:27:51 -07003616TEST_F(VideoStreamEncoderTest,
3617 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003618 const int kWidth = 1280;
3619 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003621 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003622
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003623 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003624 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003625 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003626 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003627
3628 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003629 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003630 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003631 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003632 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3633
3634 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003635 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003636 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003637 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003638 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3639
mflodmancc3d4422017-08-03 08:27:51 -07003640 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003641}
3642
mflodmancc3d4422017-08-03 08:27:51 -07003643TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003644 const int kWidth = 1280;
3645 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003647 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003648
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003649 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003650 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003651 video_stream_encoder_->SetSource(&source,
3652 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003653
3654 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3655 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003656 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3658 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3659 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3660
3661 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003662 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003663 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3666 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3667
mflodmancc3d4422017-08-03 08:27:51 -07003668 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003669}
3670
mflodmancc3d4422017-08-03 08:27:51 -07003671TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003672 const int kWidth = 1280;
3673 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003676
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003677 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003678 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003679 video_stream_encoder_->SetSource(&source,
3680 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003681
3682 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3683 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003684 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003685 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3686 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3687 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3688
3689 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003690 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003691 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003692 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3693 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3694 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3695
mflodmancc3d4422017-08-03 08:27:51 -07003696 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003697}
3698
mflodmancc3d4422017-08-03 08:27:51 -07003699TEST_F(VideoStreamEncoderTest,
3700 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003701 const int kWidth = 1280;
3702 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003704 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003705
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003706 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003707 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003708 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003709 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003710 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003711
3712 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003713 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3716 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3717
3718 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003719 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003720 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003721 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003722 EXPECT_THAT(source.sink_wants(),
3723 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003724 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3725 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3726
3727 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003728 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3731 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3733
mflodmancc3d4422017-08-03 08:27:51 -07003734 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003735}
3736
mflodmancc3d4422017-08-03 08:27:51 -07003737TEST_F(VideoStreamEncoderTest,
3738 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003739 const int kWidth = 1280;
3740 const int kHeight = 720;
3741 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003742 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003743 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003744
3745 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3746 stats.input_frame_rate = kInputFps;
3747 stats_proxy_->SetMockStats(stats);
3748
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003749 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003750 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3751 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003752 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003753
3754 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003755 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003756 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3757 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003758 EXPECT_THAT(video_source_.sink_wants(),
3759 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003760
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003761 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003762 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003763 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003764 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003765 // Give the encoder queue time to process the change in degradation preference
3766 // by waiting for an encoded frame.
3767 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3768 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003769 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003770
3771 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003772 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003773 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3774 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003775 EXPECT_THAT(new_video_source.sink_wants(),
3776 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003777
3778 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003779 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003780 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003781
mflodmancc3d4422017-08-03 08:27:51 -07003782 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003783}
3784
mflodmancc3d4422017-08-03 08:27:51 -07003785TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003786 const int kWidth = 1280;
3787 const int kHeight = 720;
3788 const size_t kNumFrames = 10;
3789
Henrik Boström381d1092020-05-12 18:49:07 +02003790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003791 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003792
asaperssond0de2952017-04-21 01:47:31 -07003793 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003794 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003795 video_source_.set_adaptation_enabled(true);
3796
3797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3798 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3799
3800 int downscales = 0;
3801 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003802 video_source_.IncomingCapturedFrame(
3803 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3804 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003805
asaperssonfab67072017-04-04 05:51:49 -07003806 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003807 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003808 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003809 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003810
3811 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3812 ++downscales;
3813
3814 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3815 EXPECT_EQ(downscales,
3816 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3817 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003818 }
mflodmancc3d4422017-08-03 08:27:51 -07003819 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003820}
3821
mflodmancc3d4422017-08-03 08:27:51 -07003822TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003823 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3824 const int kWidth = 1280;
3825 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003827 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003828
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003829 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003830 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003831 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003832 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003833 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003834
Åsa Persson8c1bf952018-09-13 10:42:19 +02003835 int64_t timestamp_ms = kFrameIntervalMs;
3836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003837 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003838 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003839 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3840 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3841
3842 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003843 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003844 timestamp_ms += kFrameIntervalMs;
3845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3846 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003847 EXPECT_THAT(source.sink_wants(),
3848 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003849 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3850 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3851
3852 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003853 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003854 timestamp_ms += kFrameIntervalMs;
3855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003856 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003857 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003858 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3859 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3860
3861 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003862 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003863 timestamp_ms += kFrameIntervalMs;
3864 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3865 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003866 EXPECT_THAT(source.sink_wants(),
3867 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003868 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3869 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3870
3871 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003872 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003873 timestamp_ms += kFrameIntervalMs;
3874 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003875 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003876 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003877 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3878 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3879
mflodmancc3d4422017-08-03 08:27:51 -07003880 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003881}
3882
mflodmancc3d4422017-08-03 08:27:51 -07003883TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003884 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3885 const int kWidth = 1280;
3886 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003888 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003889
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003890 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003891 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003892 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003893 video_stream_encoder_->SetSource(&source,
3894 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003895
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 int64_t timestamp_ms = kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003898 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003899 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003900 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3901 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3902
3903 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003904 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003905 timestamp_ms += kFrameIntervalMs;
3906 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3907 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003908 EXPECT_THAT(source.sink_wants(),
3909 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003910 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3911 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3912
3913 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003914 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003915 timestamp_ms += kFrameIntervalMs;
3916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003917 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3920 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3921
3922 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003923 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003924 timestamp_ms += kFrameIntervalMs;
3925 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3926 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003927 EXPECT_THAT(source.sink_wants(),
3928 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003929 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3930 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3931
3932 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003933 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003934 timestamp_ms += kFrameIntervalMs;
3935 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003936 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003937 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003938 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3939 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3940
mflodmancc3d4422017-08-03 08:27:51 -07003941 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003942}
3943
Sergey Silkin41c650b2019-10-14 13:12:19 +02003944TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3945 fake_encoder_.SetResolutionBitrateLimits(
3946 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3947
Henrik Boström381d1092020-05-12 18:49:07 +02003948 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003949 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3950 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3951 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3952 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003953
3954 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003955 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003956 source.set_adaptation_enabled(true);
3957 video_stream_encoder_->SetSource(
3958 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3959
3960 // Insert 720p frame.
3961 int64_t timestamp_ms = kFrameIntervalMs;
3962 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3963 WaitForEncodedFrame(1280, 720);
3964
3965 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003967 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3968 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3969 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3970 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003971 video_stream_encoder_->TriggerQualityLow();
3972
3973 // Insert 720p frame. It should be downscaled and encoded.
3974 timestamp_ms += kFrameIntervalMs;
3975 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3976 WaitForEncodedFrame(960, 540);
3977
3978 // Trigger adapt up. Higher resolution should not be requested duo to lack
3979 // of bitrate.
3980 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003981 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003982
3983 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003985 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3986 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3987 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3988 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003989
3990 // Trigger adapt up. Higher resolution should be requested.
3991 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003992 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003993
3994 video_stream_encoder_->Stop();
3995}
3996
3997TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3998 fake_encoder_.SetResolutionBitrateLimits(
3999 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4000
4001 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004003 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4004 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4005 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4006 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004007
4008 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004009 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004010 source.set_adaptation_enabled(true);
4011 video_stream_encoder_->SetSource(
4012 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4013
4014 // Insert 720p frame. It should be dropped and lower resolution should be
4015 // requested.
4016 int64_t timestamp_ms = kFrameIntervalMs;
4017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4018 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004019 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004020
4021 // Insert 720p frame. It should be downscaled and encoded.
4022 timestamp_ms += kFrameIntervalMs;
4023 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4024 WaitForEncodedFrame(960, 540);
4025
4026 video_stream_encoder_->Stop();
4027}
4028
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004029class BalancedDegradationTest : public VideoStreamEncoderTest {
4030 protected:
4031 void SetupTest() {
4032 // Reset encoder for field trials to take effect.
4033 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004034 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004035
4036 // Enable BALANCED preference.
4037 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004038 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4039 }
4040
Asa Persson606d3cb2021-10-04 10:07:11 +02004041 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004043 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004044 }
4045
Åsa Persson45b176f2019-09-30 11:19:05 +02004046 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004047 timestamp_ms_ += kFrameIntervalMs;
4048 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004049 }
4050
4051 void InsertFrameAndWaitForEncoded() {
4052 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004053 sink_.WaitForEncodedFrame(timestamp_ms_);
4054 }
4055
4056 const int kWidth = 640; // pixels:640x360=230400
4057 const int kHeight = 360;
4058 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4059 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004060 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004061};
4062
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004063TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004064 test::ScopedFieldTrials field_trials(
4065 "WebRTC-Video-BalancedDegradationSettings/"
4066 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4067 SetupTest();
4068
4069 // Force input frame rate.
4070 const int kInputFps = 24;
4071 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4072 stats.input_frame_rate = kInputFps;
4073 stats_proxy_->SetMockStats(stats);
4074
Åsa Persson45b176f2019-09-30 11:19:05 +02004075 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004076 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004077
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004078 // Trigger adapt down, expect scaled down framerate and resolution,
4079 // since Fps diff (input-requested:0) < threshold.
4080 video_stream_encoder_->TriggerQualityLow();
4081 EXPECT_THAT(source_.sink_wants(),
4082 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004083
4084 video_stream_encoder_->Stop();
4085}
4086
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004087TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004088 test::ScopedFieldTrials field_trials(
4089 "WebRTC-Video-BalancedDegradationSettings/"
4090 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4091 SetupTest();
4092
4093 // Force input frame rate.
4094 const int kInputFps = 25;
4095 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4096 stats.input_frame_rate = kInputFps;
4097 stats_proxy_->SetMockStats(stats);
4098
Åsa Persson45b176f2019-09-30 11:19:05 +02004099 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004100 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004101
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004102 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4103 // Fps diff (input-requested:1) == threshold.
4104 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004105 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004106
4107 video_stream_encoder_->Stop();
4108}
4109
4110TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4111 test::ScopedFieldTrials field_trials(
4112 "WebRTC-Video-BalancedDegradationSettings/"
4113 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4114 SetupTest();
4115
4116 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4117
Åsa Persson45b176f2019-09-30 11:19:05 +02004118 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004119 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004120
4121 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4122 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004123 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004124
4125 video_stream_encoder_->Stop();
4126}
4127
Åsa Perssonccfb3402019-09-25 15:13:04 +02004128TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004129 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004130 "WebRTC-Video-BalancedDegradationSettings/"
4131 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004132 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004133
Asa Persson606d3cb2021-10-04 10:07:11 +02004134 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4135 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4136 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004137
Åsa Persson45b176f2019-09-30 11:19:05 +02004138 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004139 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004140 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4141
4142 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4143 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004144 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004145 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004146 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4147
4148 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4149 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004150 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004151 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004152 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4153
Åsa Persson30ab0152019-08-27 12:22:33 +02004154 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4155 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004156 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004157 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004158 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004159 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4160
4161 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004162 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004163 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004164 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004165
Åsa Persson30ab0152019-08-27 12:22:33 +02004166 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004167 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004168 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004169 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004170 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004171 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4172
4173 video_stream_encoder_->Stop();
4174}
4175
Åsa Perssonccfb3402019-09-25 15:13:04 +02004176TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004177 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4178 test::ScopedFieldTrials field_trials(
4179 "WebRTC-Video-BalancedDegradationSettings/"
4180 "pixels:57600|129600|230400,fps:7|24|24/");
4181 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004182 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004183
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004184 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004185
4186 // Insert frame, expect scaled down:
4187 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4188 InsertFrame();
4189 EXPECT_FALSE(WaitForFrame(1000));
4190 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4191 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4192
4193 // Insert frame, expect scaled down:
4194 // resolution (320x180@24fps).
4195 InsertFrame();
4196 EXPECT_FALSE(WaitForFrame(1000));
4197 EXPECT_LT(source_.sink_wants().max_pixel_count,
4198 source_.last_wants().max_pixel_count);
4199 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4200
4201 // Frame should not be dropped (min pixels per frame reached).
4202 InsertFrameAndWaitForEncoded();
4203
4204 video_stream_encoder_->Stop();
4205}
4206
4207TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004208 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004209 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004210 "WebRTC-Video-BalancedDegradationSettings/"
4211 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004212 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004213
Asa Persson606d3cb2021-10-04 10:07:11 +02004214 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4215 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4216 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004217
Åsa Persson45b176f2019-09-30 11:19:05 +02004218 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004219 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004220 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4221
4222 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4223 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004224 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004225 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004226 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4227
4228 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4229 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004230 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004231 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004232 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4233
4234 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4235 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004236 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004237 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004238 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4239
Åsa Persson30ab0152019-08-27 12:22:33 +02004240 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4241 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004242 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004243 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004244 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4245
4246 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4247 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004248 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004249 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4250
4251 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004252 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004253 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004255 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004256 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4257
4258 video_stream_encoder_->Stop();
4259}
4260
Åsa Perssonccfb3402019-09-25 15:13:04 +02004261TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004262 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004263 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004264 "WebRTC-Video-BalancedDegradationSettings/"
4265 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004266 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004267
Asa Persson606d3cb2021-10-04 10:07:11 +02004268 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4269 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4270 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4271 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4272 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004273
Åsa Persson45b176f2019-09-30 11:19:05 +02004274 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004275 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004276 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4277
4278 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4279 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004280 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004281 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004282 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4283
4284 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4285 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004286 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004287 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004288 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4289
4290 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4291 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004292 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004293 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004294 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4295
4296 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4297 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004298 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004299 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4300
4301 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004302 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004303 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004304 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004305 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004306 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4307
4308 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004309 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004310 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004311 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004312 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
4314 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004315 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004316 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004317 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004318 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004319 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4320
Åsa Persson1b247f12019-08-14 17:26:39 +02004321 video_stream_encoder_->Stop();
4322}
4323
mflodmancc3d4422017-08-03 08:27:51 -07004324TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004325 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4326 const int kWidth = 1280;
4327 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004329 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004330
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004331 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004332 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004333 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004334 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004335 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004336
Åsa Persson8c1bf952018-09-13 10:42:19 +02004337 int64_t timestamp_ms = kFrameIntervalMs;
4338 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004339 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004340 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004341 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4342 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4343 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4344 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004347 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004348 timestamp_ms += kFrameIntervalMs;
4349 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4350 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004351 EXPECT_THAT(source.sink_wants(),
4352 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004353 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4354 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4355 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4356 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4357
4358 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004359 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004360 timestamp_ms += kFrameIntervalMs;
4361 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4362 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004363 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004364 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4366 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4367 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4368
Jonathan Yubc771b72017-12-08 17:04:29 -08004369 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004370 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004371 timestamp_ms += kFrameIntervalMs;
4372 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4373 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004374 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004375 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4376 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004377 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004378 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4379
Jonathan Yubc771b72017-12-08 17:04:29 -08004380 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004381 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004382 timestamp_ms += kFrameIntervalMs;
4383 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4384 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004385 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004386 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004387 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4388 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4389 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4391
Jonathan Yubc771b72017-12-08 17:04:29 -08004392 // Trigger quality adapt down, expect no change (min resolution reached).
4393 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004394 timestamp_ms += kFrameIntervalMs;
4395 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4396 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004397 EXPECT_THAT(source.sink_wants(), FpsMax());
4398 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004399 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4401 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4402 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4403
Evan Shrubsole64469032020-06-11 10:45:29 +02004404 // Trigger quality adapt up, expect upscaled resolution (480x270).
4405 video_stream_encoder_->TriggerQualityHigh();
4406 timestamp_ms += kFrameIntervalMs;
4407 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4408 WaitForEncodedFrame(timestamp_ms);
4409 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4410 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4411 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4412 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4413 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4414
4415 // Trigger quality and cpu adapt up since both are most limited, expect
4416 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004417 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004418 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004419 timestamp_ms += kFrameIntervalMs;
4420 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4421 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004422 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004423 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4425 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004426 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004427
Evan Shrubsole64469032020-06-11 10:45:29 +02004428 // Trigger quality and cpu adapt up since both are most limited, expect
4429 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004430 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004431 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004432 timestamp_ms += kFrameIntervalMs;
4433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4434 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004435 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004436 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004437 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004439 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4440 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004441
Evan Shrubsole64469032020-06-11 10:45:29 +02004442 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4443 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004444 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004445 timestamp_ms += kFrameIntervalMs;
4446 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4447 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004448 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004449 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4450 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004451 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004452 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004453
4454 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004455 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004456 timestamp_ms += kFrameIntervalMs;
4457 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004458 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004459 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004460 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004461 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4462 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004463 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004464 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004465
mflodmancc3d4422017-08-03 08:27:51 -07004466 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004467}
4468
mflodmancc3d4422017-08-03 08:27:51 -07004469TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004470 const int kWidth = 640;
4471 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004472
Henrik Boström381d1092020-05-12 18:49:07 +02004473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004474 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004475
perkj803d97f2016-11-01 11:45:46 -07004476 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004477 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004478 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004479 }
4480
mflodmancc3d4422017-08-03 08:27:51 -07004481 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004482 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004483 video_source_.IncomingCapturedFrame(CreateFrame(
4484 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004485 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004486 }
4487
mflodmancc3d4422017-08-03 08:27:51 -07004488 video_stream_encoder_->Stop();
4489 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004490 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004491
Ying Wangef3998f2019-12-09 13:06:53 +01004492 EXPECT_METRIC_EQ(
4493 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4494 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004495 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4496}
4497
mflodmancc3d4422017-08-03 08:27:51 -07004498TEST_F(VideoStreamEncoderTest,
4499 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004500 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004501 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004502 const int kWidth = 640;
4503 const int kHeight = 360;
4504
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004505 video_stream_encoder_->SetSource(&video_source_,
4506 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004507
4508 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4509 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004510 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004511 }
4512
mflodmancc3d4422017-08-03 08:27:51 -07004513 video_stream_encoder_->Stop();
4514 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004515 stats_proxy_.reset();
4516
4517 EXPECT_EQ(0,
4518 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4519}
4520
Per Kjellanderdcef6412020-10-07 15:09:05 +02004521TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4522 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004523 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004524 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004525
4526 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004527 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004528 SimulcastRateAllocator(fake_encoder_.config())
4529 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004530 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004531
Henrik Boström381d1092020-05-12 18:49:07 +02004532 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004533 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004534
sprang57c2fff2017-01-16 06:24:02 -08004535 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004536 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4537 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004538 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4539 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4540
Erik Språngd7329ca2019-02-21 21:19:53 +01004541 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004542 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004543 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004544
Per Kjellanderdcef6412020-10-07 15:09:05 +02004545 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004546 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004547 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4548 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004549 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004550 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004551
Per Kjellanderdcef6412020-10-07 15:09:05 +02004552 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004553 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004554 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004555 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004556 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4557 WaitForEncodedFrame(CurrentTimeMs());
4558 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004559 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004560 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004561
mflodmancc3d4422017-08-03 08:27:51 -07004562 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004563}
4564
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004565TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004566 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004567 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004568 kVideoLayersAllocation);
4569
4570 const int kDefaultFps = 30;
4571
4572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004573 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004574
4575 video_source_.IncomingCapturedFrame(
4576 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4577 WaitForEncodedFrame(CurrentTimeMs());
4578 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4579 VideoLayersAllocation last_layer_allocation =
4580 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004581 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004582 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4583
4584 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004585 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004586 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004587 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004588 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4589
Erik Språng9d69cbe2020-10-22 17:44:42 +02004590 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004591 int number_of_layers_allocation = 1;
4592 const int64_t start_time_ms = CurrentTimeMs();
4593 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4594 video_source_.IncomingCapturedFrame(
4595 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4596 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004597 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4598 number_of_layers_allocation = sink_.number_of_layers_allocations();
4599 VideoLayersAllocation new_allocation =
4600 sink_.GetLastVideoLayersAllocation();
4601 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4602 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4603 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4604 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4605 .target_bitrate_per_temporal_layer,
4606 last_layer_allocation.active_spatial_layers[0]
4607 .target_bitrate_per_temporal_layer);
4608 last_layer_allocation = new_allocation;
4609 }
4610 }
4611 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4612 video_stream_encoder_->Stop();
4613}
4614
4615TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004616 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004617 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4618 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4619 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004620 VideoEncoderConfig video_encoder_config;
4621 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4622 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004623 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004624 video_encoder_config.content_type =
4625 VideoEncoderConfig::ContentType::kRealtimeVideo;
4626 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004627 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004628 VideoEncoder::GetDefaultVp8Settings());
4629 for (auto& layer : video_encoder_config.simulcast_layers) {
4630 layer.num_temporal_layers = 2;
4631 }
4632 // Simulcast layers are used for enabling/disabling streams.
4633 video_encoder_config.simulcast_layers[0].active = true;
4634 video_encoder_config.simulcast_layers[1].active = false;
4635 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004636 ConfigureEncoder(std::move(video_encoder_config),
4637 VideoStreamEncoder::BitrateAllocationCallbackType::
4638 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004639
4640 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004641 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004642
4643 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4644 WaitForEncodedFrame(CurrentTimeMs());
4645 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4646 VideoLayersAllocation last_layer_allocation =
4647 sink_.GetLastVideoLayersAllocation();
4648
4649 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4650 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4651 .target_bitrate_per_temporal_layer,
4652 SizeIs(2));
4653 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4654 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4655 video_stream_encoder_->Stop();
4656}
4657
4658TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004659 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004660 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4661 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4662 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004663 VideoEncoderConfig video_encoder_config;
4664 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4665 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004666 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004667 video_encoder_config.content_type =
4668 VideoEncoderConfig::ContentType::kRealtimeVideo;
4669 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004670 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004671 VideoEncoder::GetDefaultVp8Settings());
4672 for (auto& layer : video_encoder_config.simulcast_layers) {
4673 layer.num_temporal_layers = 2;
4674 }
4675 // Simulcast layers are used for enabling/disabling streams.
4676 video_encoder_config.simulcast_layers[0].active = true;
4677 video_encoder_config.simulcast_layers[1].active = false;
4678 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004679 ConfigureEncoder(std::move(video_encoder_config),
4680 VideoStreamEncoder::BitrateAllocationCallbackType::
4681 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004682
4683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004685
4686 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4687 WaitForEncodedFrame(CurrentTimeMs());
4688 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4689 VideoLayersAllocation last_layer_allocation =
4690 sink_.GetLastVideoLayersAllocation();
4691
4692 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4693 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4694 .target_bitrate_per_temporal_layer,
4695 SizeIs(2));
4696 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4697
4698 video_stream_encoder_->Stop();
4699}
4700
4701TEST_F(VideoStreamEncoderTest,
4702 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4703 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4704 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004705 VideoEncoderConfig video_encoder_config;
4706 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4707 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004708 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004709 video_encoder_config.content_type =
4710 VideoEncoderConfig::ContentType::kRealtimeVideo;
4711 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4712 vp9_settings.numberOfSpatialLayers = 2;
4713 vp9_settings.numberOfTemporalLayers = 2;
4714 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4715 vp9_settings.automaticResizeOn = false;
4716 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004717 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004719 ConfigureEncoder(std::move(video_encoder_config),
4720 VideoStreamEncoder::BitrateAllocationCallbackType::
4721 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004722
4723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725
4726 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4727 WaitForEncodedFrame(CurrentTimeMs());
4728 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4729 VideoLayersAllocation last_layer_allocation =
4730 sink_.GetLastVideoLayersAllocation();
4731
4732 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4733 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4734 .target_bitrate_per_temporal_layer,
4735 SizeIs(2));
4736 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4737 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4738 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4739 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4740 .target_bitrate_per_temporal_layer,
4741 SizeIs(2));
4742 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4743 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4744 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4745
4746 // Since full SVC is used, expect the top layer to utilize the full target
4747 // rate.
4748 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4749 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004750 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004751 video_stream_encoder_->Stop();
4752}
4753
4754TEST_F(VideoStreamEncoderTest,
4755 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4756 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4757 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004758 VideoEncoderConfig video_encoder_config;
4759 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4760 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004761 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004762 video_encoder_config.content_type =
4763 VideoEncoderConfig::ContentType::kRealtimeVideo;
4764 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4765 vp9_settings.numberOfSpatialLayers = 2;
4766 vp9_settings.numberOfTemporalLayers = 2;
4767 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4768 vp9_settings.automaticResizeOn = false;
4769 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004770 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004771 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004772 ConfigureEncoder(std::move(video_encoder_config),
4773 VideoStreamEncoder::BitrateAllocationCallbackType::
4774 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775
4776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004777 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004778
4779 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4780 WaitForEncodedFrame(CurrentTimeMs());
4781 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4782 VideoLayersAllocation last_layer_allocation =
4783 sink_.GetLastVideoLayersAllocation();
4784
4785 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4786 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4787 .target_bitrate_per_temporal_layer,
4788 SizeIs(1));
4789 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4790 .target_bitrate_per_temporal_layer,
4791 SizeIs(1));
4792 // Since full SVC is used, expect the top layer to utilize the full target
4793 // rate.
4794 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4795 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004796 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004797 video_stream_encoder_->Stop();
4798}
4799
4800TEST_F(VideoStreamEncoderTest,
4801 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4802 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4803 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004804 VideoEncoderConfig video_encoder_config;
4805 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4806 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004807 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004808 video_encoder_config.content_type =
4809 VideoEncoderConfig::ContentType::kRealtimeVideo;
4810 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4811 vp9_settings.numberOfSpatialLayers = 2;
4812 vp9_settings.numberOfTemporalLayers = 2;
4813 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4814 vp9_settings.automaticResizeOn = false;
4815 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004816 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004817 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004818 ConfigureEncoder(std::move(video_encoder_config),
4819 VideoStreamEncoder::BitrateAllocationCallbackType::
4820 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821
4822 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004823 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004824
4825 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4826 WaitForEncodedFrame(CurrentTimeMs());
4827 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4828 VideoLayersAllocation last_layer_allocation =
4829 sink_.GetLastVideoLayersAllocation();
4830
4831 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4832 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4833 .target_bitrate_per_temporal_layer,
4834 SizeIs(2));
4835 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4836 .target_bitrate_per_temporal_layer,
4837 SizeIs(2));
4838 // Since KSVC is, spatial layers are independend except on key frames.
4839 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4840 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004841 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004842 video_stream_encoder_->Stop();
4843}
4844
4845TEST_F(VideoStreamEncoderTest,
4846 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4847 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4848 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4849 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004850 VideoEncoderConfig video_encoder_config;
4851 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4852 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004853 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004854 video_encoder_config.content_type =
4855 VideoEncoderConfig::ContentType::kRealtimeVideo;
4856 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4857 vp9_settings.numberOfSpatialLayers = 3;
4858 vp9_settings.numberOfTemporalLayers = 2;
4859 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4860 vp9_settings.automaticResizeOn = false;
4861 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004862 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004863 vp9_settings);
4864 // Simulcast layers are used for enabling/disabling streams.
4865 video_encoder_config.simulcast_layers.resize(3);
4866 video_encoder_config.simulcast_layers[0].active = false;
4867 video_encoder_config.simulcast_layers[1].active = true;
4868 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004869 ConfigureEncoder(std::move(video_encoder_config),
4870 VideoStreamEncoder::BitrateAllocationCallbackType::
4871 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004872
4873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004874 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004875
4876 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4877 WaitForEncodedFrame(CurrentTimeMs());
4878 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4879 VideoLayersAllocation last_layer_allocation =
4880 sink_.GetLastVideoLayersAllocation();
4881
4882 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4883 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4884 .target_bitrate_per_temporal_layer,
4885 SizeIs(2));
4886 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4887 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4888
4889 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4890 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4891 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4892 .target_bitrate_per_temporal_layer,
4893 SizeIs(2));
4894 // Since full SVC is used, expect the top layer to utilize the full target
4895 // rate.
4896 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4897 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004898 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004899 video_stream_encoder_->Stop();
4900}
4901
4902TEST_F(VideoStreamEncoderTest,
4903 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4904 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4906 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004907 VideoEncoderConfig video_encoder_config;
4908 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4909 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004910 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004911 video_encoder_config.content_type =
4912 VideoEncoderConfig::ContentType::kRealtimeVideo;
4913 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4914 vp9_settings.numberOfSpatialLayers = 3;
4915 vp9_settings.numberOfTemporalLayers = 2;
4916 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4917 vp9_settings.automaticResizeOn = false;
4918 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004919 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920 vp9_settings);
4921 // Simulcast layers are used for enabling/disabling streams.
4922 video_encoder_config.simulcast_layers.resize(3);
4923 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004924 ConfigureEncoder(std::move(video_encoder_config),
4925 VideoStreamEncoder::BitrateAllocationCallbackType::
4926 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004927
4928 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004929 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004930
4931 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4932 WaitForEncodedFrame(CurrentTimeMs());
4933 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4934 VideoLayersAllocation last_layer_allocation =
4935 sink_.GetLastVideoLayersAllocation();
4936
4937 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4938 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4939 .target_bitrate_per_temporal_layer,
4940 SizeIs(2));
4941 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4942 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4943
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4946 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4947 .target_bitrate_per_temporal_layer,
4948 SizeIs(2));
4949 video_stream_encoder_->Stop();
4950}
4951
4952TEST_F(VideoStreamEncoderTest,
4953 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4954 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4955 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4956 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004957 VideoEncoderConfig video_encoder_config;
4958 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4959 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004960 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004961 video_encoder_config.content_type =
4962 VideoEncoderConfig::ContentType::kRealtimeVideo;
4963 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4964 vp9_settings.numberOfSpatialLayers = 3;
4965 vp9_settings.numberOfTemporalLayers = 2;
4966 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4967 vp9_settings.automaticResizeOn = false;
4968 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004969 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004970 vp9_settings);
4971 // Simulcast layers are used for enabling/disabling streams.
4972 video_encoder_config.simulcast_layers.resize(3);
4973 video_encoder_config.simulcast_layers[0].active = false;
4974 video_encoder_config.simulcast_layers[1].active = false;
4975 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004976 ConfigureEncoder(std::move(video_encoder_config),
4977 VideoStreamEncoder::BitrateAllocationCallbackType::
4978 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004979
4980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004981 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004982
4983 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4984 WaitForEncodedFrame(CurrentTimeMs());
4985 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4986 VideoLayersAllocation last_layer_allocation =
4987 sink_.GetLastVideoLayersAllocation();
4988
4989 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4990 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4991 .target_bitrate_per_temporal_layer,
4992 SizeIs(2));
4993 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4994 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4995 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4996 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004997 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004998 video_stream_encoder_->Stop();
4999}
5000
5001TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5002 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005003 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005004 kVideoLayersAllocation);
5005 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005006 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005007
5008 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5009 WaitForEncodedFrame(CurrentTimeMs());
5010 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5011 VideoLayersAllocation last_layer_allocation =
5012 sink_.GetLastVideoLayersAllocation();
5013
5014 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5015 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5016 .target_bitrate_per_temporal_layer,
5017 SizeIs(1));
5018 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5019 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005020 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005021 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5022 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5023 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5024 video_stream_encoder_->Stop();
5025}
5026
5027TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005028 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5029 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005030 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005031 kVideoLayersAllocation);
5032
5033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005034 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005035
5036 video_source_.IncomingCapturedFrame(
5037 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5038 WaitForEncodedFrame(CurrentTimeMs());
5039 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5040 VideoLayersAllocation last_layer_allocation =
5041 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005042 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005043 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5044 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5045 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005046 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005047
5048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005049 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5050 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005051 video_source_.IncomingCapturedFrame(
5052 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5053 WaitForEncodedFrame(CurrentTimeMs());
5054
5055 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5056 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5057 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5058 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5059 .target_bitrate_per_temporal_layer[0],
5060 DataRate::Zero());
5061
5062 video_stream_encoder_->Stop();
5063}
5064
Per Kjellander4190ce92020-12-15 17:24:55 +01005065TEST_F(VideoStreamEncoderTest,
5066 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5067 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005068 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005069 kVideoLayersAllocation);
5070
5071 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005072 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5073 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005074
5075 video_source_.IncomingCapturedFrame(
5076 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5077 WaitForEncodedFrame(CurrentTimeMs());
5078 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5079 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5080 SizeIs(2));
5081 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5082 codec_width_);
5083 EXPECT_EQ(
5084 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5085 codec_height_);
5086
5087 video_source_.IncomingCapturedFrame(
5088 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5089 WaitForEncodedFrame(CurrentTimeMs());
5090 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5091 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5092 SizeIs(2));
5093 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5094 codec_width_ / 2);
5095 EXPECT_EQ(
5096 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5097 codec_height_ / 2);
5098
5099 video_stream_encoder_->Stop();
5100}
5101
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005102TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5103 // 2 TLs configured, temporal layers supported by encoder.
5104 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005105 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005106 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005107 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005108 fake_encoder_.SetTemporalLayersSupported(0, true);
5109
5110 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005111 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005112 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005113 kNumTemporalLayers, /*temporal_id*/ 0,
5114 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005115 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005116 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005117 kNumTemporalLayers, /*temporal_id*/ 1,
5118 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005119 VideoBitrateAllocation expected_bitrate;
5120 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5121 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5122
5123 VerifyAllocatedBitrate(expected_bitrate);
5124 video_stream_encoder_->Stop();
5125}
5126
5127TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5128 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005129 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005130 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005131 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005132 fake_encoder_.SetTemporalLayersSupported(0, false);
5133
5134 // Temporal layers not supported by the encoder.
5135 // Total bitrate should be at ti:0.
5136 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005137 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005138
5139 VerifyAllocatedBitrate(expected_bitrate);
5140 video_stream_encoder_->Stop();
5141}
5142
5143TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005144 webrtc::test::ScopedFieldTrials field_trials(
5145 "WebRTC-Video-QualityScalerSettings/"
5146 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5147 // Reset encoder for field trials to take effect.
5148 ConfigureEncoder(video_encoder_config_.Copy());
5149
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005150 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005151 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005152 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005153 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005154 fake_encoder_.SetTemporalLayersSupported(0, true);
5155 fake_encoder_.SetTemporalLayersSupported(1, false);
5156
5157 const int kS0Bps = 150000;
5158 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005159 kS0Bps *
5160 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5161 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005162 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005163 kS0Bps *
5164 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5165 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005166 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005167 // Temporal layers not supported by si:1.
5168 VideoBitrateAllocation expected_bitrate;
5169 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5170 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5171 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5172
5173 VerifyAllocatedBitrate(expected_bitrate);
5174 video_stream_encoder_->Stop();
5175}
5176
Niels Möller7dc26b72017-12-06 10:27:48 +01005177TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5178 const int kFrameWidth = 1280;
5179 const int kFrameHeight = 720;
5180 const int kFramerate = 24;
5181
Henrik Boström381d1092020-05-12 18:49:07 +02005182 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005183 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005184 test::FrameForwarder source;
5185 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005186 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005187
5188 // Insert a single frame, triggering initial configuration.
5189 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5190 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5191
5192 EXPECT_EQ(
5193 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5194 kDefaultFramerate);
5195
5196 // Trigger reconfigure encoder (without resetting the entire instance).
5197 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005198 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5199 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005200 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005201 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005202 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005203 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5204
5205 // Detector should be updated with fps limit from codec config.
5206 EXPECT_EQ(
5207 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5208 kFramerate);
5209
5210 // Trigger overuse, max framerate should be reduced.
5211 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5212 stats.input_frame_rate = kFramerate;
5213 stats_proxy_->SetMockStats(stats);
5214 video_stream_encoder_->TriggerCpuOveruse();
5215 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5216 int adapted_framerate =
5217 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5218 EXPECT_LT(adapted_framerate, kFramerate);
5219
5220 // Trigger underuse, max framerate should go back to codec configured fps.
5221 // Set extra low fps, to make sure it's actually reset, not just incremented.
5222 stats = stats_proxy_->GetStats();
5223 stats.input_frame_rate = adapted_framerate / 2;
5224 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005225 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005226 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5227 EXPECT_EQ(
5228 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5229 kFramerate);
5230
5231 video_stream_encoder_->Stop();
5232}
5233
5234TEST_F(VideoStreamEncoderTest,
5235 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5236 const int kFrameWidth = 1280;
5237 const int kFrameHeight = 720;
5238 const int kLowFramerate = 15;
5239 const int kHighFramerate = 25;
5240
Henrik Boström381d1092020-05-12 18:49:07 +02005241 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005242 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005243 test::FrameForwarder source;
5244 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005245 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005246
5247 // Trigger initial configuration.
5248 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005249 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5250 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005251 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005252 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005253 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005254 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005255 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5256
5257 EXPECT_EQ(
5258 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5259 kLowFramerate);
5260
5261 // Trigger overuse, max framerate should be reduced.
5262 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5263 stats.input_frame_rate = kLowFramerate;
5264 stats_proxy_->SetMockStats(stats);
5265 video_stream_encoder_->TriggerCpuOveruse();
5266 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5267 int adapted_framerate =
5268 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5269 EXPECT_LT(adapted_framerate, kLowFramerate);
5270
5271 // Reconfigure the encoder with a new (higher max framerate), max fps should
5272 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005273 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005274 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5275 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005276 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005277 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5278
5279 EXPECT_EQ(
5280 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5281 adapted_framerate);
5282
5283 // Trigger underuse, max framerate should go back to codec configured fps.
5284 stats = stats_proxy_->GetStats();
5285 stats.input_frame_rate = adapted_framerate;
5286 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005287 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005288 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5289 EXPECT_EQ(
5290 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5291 kHighFramerate);
5292
5293 video_stream_encoder_->Stop();
5294}
5295
mflodmancc3d4422017-08-03 08:27:51 -07005296TEST_F(VideoStreamEncoderTest,
5297 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005298 const int kFrameWidth = 1280;
5299 const int kFrameHeight = 720;
5300 const int kFramerate = 24;
5301
Henrik Boström381d1092020-05-12 18:49:07 +02005302 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005303 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005304 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005305 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005306 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005307
5308 // Trigger initial configuration.
5309 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005310 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5311 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005312 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005313 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005314 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005315 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005316 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005317
Niels Möller7dc26b72017-12-06 10:27:48 +01005318 EXPECT_EQ(
5319 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5320 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005321
5322 // Trigger overuse, max framerate should be reduced.
5323 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5324 stats.input_frame_rate = kFramerate;
5325 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005326 video_stream_encoder_->TriggerCpuOveruse();
5327 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005328 int adapted_framerate =
5329 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005330 EXPECT_LT(adapted_framerate, kFramerate);
5331
5332 // Change degradation preference to not enable framerate scaling. Target
5333 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005334 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005335 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005336 EXPECT_EQ(
5337 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5338 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005339
mflodmancc3d4422017-08-03 08:27:51 -07005340 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005341}
5342
mflodmancc3d4422017-08-03 08:27:51 -07005343TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005344 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005346 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5347 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5348 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005349 const int kWidth = 640;
5350 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005351
asaperssonfab67072017-04-04 05:51:49 -07005352 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005353
5354 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005355 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005356
5357 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005358 EXPECT_TRUE_WAIT(
5359 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005360
sprangc5d62e22017-04-02 23:53:04 -07005361 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005362
asaperssonfab67072017-04-04 05:51:49 -07005363 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005364 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005365 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005366
5367 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005368 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005369
Henrik Boström2671dac2020-05-19 16:29:09 +02005370 EXPECT_TRUE_WAIT(
5371 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005372
mflodmancc3d4422017-08-03 08:27:51 -07005373 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005374}
5375
mflodmancc3d4422017-08-03 08:27:51 -07005376TEST_F(VideoStreamEncoderTest,
5377 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005378 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005379 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005380 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5381 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5382 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005383 const int kWidth = 640;
5384 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005385
5386 // We expect the n initial frames to get dropped.
5387 int i;
5388 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005389 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005390 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005391 }
5392 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005393 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005394 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005395
5396 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005397 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005398
mflodmancc3d4422017-08-03 08:27:51 -07005399 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005400}
5401
mflodmancc3d4422017-08-03 08:27:51 -07005402TEST_F(VideoStreamEncoderTest,
5403 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005404 const int kWidth = 640;
5405 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005407 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005408
5409 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005410 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005411 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005412
asaperssonfab67072017-04-04 05:51:49 -07005413 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005414 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005415 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005416
mflodmancc3d4422017-08-03 08:27:51 -07005417 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005418}
5419
mflodmancc3d4422017-08-03 08:27:51 -07005420TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005421 const int kWidth = 640;
5422 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005423 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005424
5425 VideoEncoderConfig video_encoder_config;
5426 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5427 // Make format different, to force recreation of encoder.
5428 video_encoder_config.video_format.parameters["foo"] = "foo";
5429 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005430 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005431 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005432 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005433
kthelgasonb83797b2017-02-14 11:57:25 -08005434 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005435 video_stream_encoder_->SetSource(&video_source_,
5436 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005437
asaperssonfab67072017-04-04 05:51:49 -07005438 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005439 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005440 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005441
mflodmancc3d4422017-08-03 08:27:51 -07005442 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005443 fake_encoder_.SetQualityScaling(true);
5444}
5445
Åsa Persson139f4dc2019-08-02 09:29:58 +02005446TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5447 webrtc::test::ScopedFieldTrials field_trials(
5448 "WebRTC-Video-QualityScalerSettings/"
5449 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5450 // Reset encoder for field trials to take effect.
5451 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005452 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5453 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005454 const int kWidth = 640;
5455 const int kHeight = 360;
5456
Henrik Boström381d1092020-05-12 18:49:07 +02005457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005458 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005459 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5460 // Frame should not be dropped.
5461 WaitForEncodedFrame(1);
5462
Henrik Boström381d1092020-05-12 18:49:07 +02005463 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005464 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5465 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5466 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005467 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5468 // Frame should not be dropped.
5469 WaitForEncodedFrame(2);
5470
Henrik Boström381d1092020-05-12 18:49:07 +02005471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005472 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5473 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5474 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005475 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5476 // Expect to drop this frame, the wait should time out.
5477 ExpectDroppedFrame();
5478
5479 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005480 EXPECT_TRUE_WAIT(
5481 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005482 video_stream_encoder_->Stop();
5483}
5484
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005485TEST_F(VideoStreamEncoderTest,
5486 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5487 webrtc::test::ScopedFieldTrials field_trials(
5488 "WebRTC-Video-QualityScalerSettings/"
5489 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5490 fake_encoder_.SetQualityScaling(false);
5491 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005492 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5493 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005494 const int kWidth = 640;
5495 const int kHeight = 360;
5496
5497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005498 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005499 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5500 // Frame should not be dropped.
5501 WaitForEncodedFrame(1);
5502
5503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5504 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5505 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5506 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5507 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5508 // Frame should not be dropped.
5509 WaitForEncodedFrame(2);
5510
5511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5512 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5513 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5514 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5515 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5516 // Not dropped since quality scaling is disabled.
5517 WaitForEncodedFrame(3);
5518
5519 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005520 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005521 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5522
5523 video_stream_encoder_->Stop();
5524}
5525
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005526TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005527 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005528 // Set simulcast.
5529 ResetEncoder("VP8", 3, 1, 1, false);
5530 fake_encoder_.SetQualityScaling(true);
5531 const int kWidth = 1280;
5532 const int kHeight = 720;
5533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005534 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005535 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5536 // Frame should not be dropped.
5537 WaitForEncodedFrame(1);
5538
5539 // Trigger QVGA "singlecast"
5540 // Update the config.
5541 VideoEncoderConfig video_encoder_config;
5542 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5543 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005544 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005545 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005546 "VP8", /*max qp*/ 56, /*screencast*/ false,
5547 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005548 for (auto& layer : video_encoder_config.simulcast_layers) {
5549 layer.num_temporal_layers = 1;
5550 layer.max_framerate = kDefaultFramerate;
5551 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005552 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005553 video_encoder_config.content_type =
5554 VideoEncoderConfig::ContentType::kRealtimeVideo;
5555
5556 video_encoder_config.simulcast_layers[0].active = true;
5557 video_encoder_config.simulcast_layers[1].active = false;
5558 video_encoder_config.simulcast_layers[2].active = false;
5559
5560 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5561 kMaxPayloadLength);
5562 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5563
5564 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5565 // Frame should not be dropped.
5566 WaitForEncodedFrame(2);
5567
5568 // Trigger HD "singlecast"
5569 video_encoder_config.simulcast_layers[0].active = false;
5570 video_encoder_config.simulcast_layers[1].active = false;
5571 video_encoder_config.simulcast_layers[2].active = true;
5572
5573 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5574 kMaxPayloadLength);
5575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5576
5577 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5578 // Frame should be dropped because of initial frame drop.
5579 ExpectDroppedFrame();
5580
5581 // Expect the sink_wants to specify a scaled frame.
5582 EXPECT_TRUE_WAIT(
5583 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5584 video_stream_encoder_->Stop();
5585}
5586
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005587TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005588 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005589 // Set simulcast.
5590 ResetEncoder("VP9", 1, 1, 3, false);
5591 fake_encoder_.SetQualityScaling(true);
5592 const int kWidth = 1280;
5593 const int kHeight = 720;
5594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005595 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005596 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5597 // Frame should not be dropped.
5598 WaitForEncodedFrame(1);
5599
5600 // Trigger QVGA "singlecast"
5601 // Update the config.
5602 VideoEncoderConfig video_encoder_config;
5603 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5604 &video_encoder_config);
5605 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5606 vp9_settings.numberOfSpatialLayers = 3;
5607 // Since only one layer is active - automatic resize should be enabled.
5608 vp9_settings.automaticResizeOn = true;
5609 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005610 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005611 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005612 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005613 video_encoder_config.content_type =
5614 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005615 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005616 // which SVC layers are active.
5617 video_encoder_config.simulcast_layers.resize(3);
5618
5619 video_encoder_config.simulcast_layers[0].active = true;
5620 video_encoder_config.simulcast_layers[1].active = false;
5621 video_encoder_config.simulcast_layers[2].active = false;
5622
5623 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5624 kMaxPayloadLength);
5625 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5626
5627 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5628 // Frame should not be dropped.
5629 WaitForEncodedFrame(2);
5630
5631 // Trigger HD "singlecast"
5632 video_encoder_config.simulcast_layers[0].active = false;
5633 video_encoder_config.simulcast_layers[1].active = false;
5634 video_encoder_config.simulcast_layers[2].active = true;
5635
5636 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5637 kMaxPayloadLength);
5638 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5639
5640 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5641 // Frame should be dropped because of initial frame drop.
5642 ExpectDroppedFrame();
5643
5644 // Expect the sink_wants to specify a scaled frame.
5645 EXPECT_TRUE_WAIT(
5646 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5647 video_stream_encoder_->Stop();
5648}
5649
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005650TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005651 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5652 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5653 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5654 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5655 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5656 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5657 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5658 fake_encoder_.SetResolutionBitrateLimits(
5659 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5660
5661 VideoEncoderConfig video_encoder_config;
5662 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5663 &video_encoder_config);
5664 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5665 vp9_settings.numberOfSpatialLayers = 3;
5666 // Since only one layer is active - automatic resize should be enabled.
5667 vp9_settings.automaticResizeOn = true;
5668 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005669 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005670 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005671 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005672 video_encoder_config.content_type =
5673 VideoEncoderConfig::ContentType::kRealtimeVideo;
5674 // Simulcast layers are used to indicate which spatial layers are active.
5675 video_encoder_config.simulcast_layers.resize(3);
5676 video_encoder_config.simulcast_layers[0].active = false;
5677 video_encoder_config.simulcast_layers[1].active = true;
5678 video_encoder_config.simulcast_layers[2].active = false;
5679
5680 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5681 kMaxPayloadLength);
5682 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5683
5684 // The encoder bitrate limits for 360p should be used.
5685 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5686 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005687 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5688 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5689 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5690 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5691 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5692 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005693 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005694 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005695 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005696 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005697
5698 // The encoder bitrate limits for 270p should be used.
5699 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5700 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005701 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5702 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5703 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5704 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5705 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5706 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005707 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005708 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005709 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005710 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005711
5712 video_stream_encoder_->Stop();
5713}
5714
5715TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005716 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5717 VideoEncoderConfig video_encoder_config;
5718 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5719 &video_encoder_config);
5720 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5721 vp9_settings.numberOfSpatialLayers = 3;
5722 // Since only one layer is active - automatic resize should be enabled.
5723 vp9_settings.automaticResizeOn = true;
5724 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005725 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005726 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005727 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005728 video_encoder_config.content_type =
5729 VideoEncoderConfig::ContentType::kRealtimeVideo;
5730 // Simulcast layers are used to indicate which spatial layers are active.
5731 video_encoder_config.simulcast_layers.resize(3);
5732 video_encoder_config.simulcast_layers[0].active = false;
5733 video_encoder_config.simulcast_layers[1].active = true;
5734 video_encoder_config.simulcast_layers[2].active = false;
5735
5736 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5737 kMaxPayloadLength);
5738 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5739
5740 // The default bitrate limits for 360p should be used.
5741 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005742 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5743 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005744 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5745 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005746 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5747 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5748 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5749 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5750 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5751 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005752 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005753 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005754 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005755 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005756
5757 // The default bitrate limits for 270p should be used.
5758 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005759 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5760 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005761 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5762 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005763 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5764 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5765 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5766 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5767 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5768 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005769 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005770 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005771 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005772 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005773
5774 video_stream_encoder_->Stop();
5775}
5776
5777TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5778 webrtc::test::ScopedFieldTrials field_trials(
5779 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5780 VideoEncoderConfig video_encoder_config;
5781 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5782 &video_encoder_config);
5783 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5784 vp9_settings.numberOfSpatialLayers = 3;
5785 // Since only one layer is active - automatic resize should be enabled.
5786 vp9_settings.automaticResizeOn = true;
5787 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005788 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005789 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005790 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005791 video_encoder_config.content_type =
5792 VideoEncoderConfig::ContentType::kRealtimeVideo;
5793 // Simulcast layers are used to indicate which spatial layers are active.
5794 video_encoder_config.simulcast_layers.resize(3);
5795 video_encoder_config.simulcast_layers[0].active = false;
5796 video_encoder_config.simulcast_layers[1].active = true;
5797 video_encoder_config.simulcast_layers[2].active = false;
5798
5799 // Reset encoder for field trials to take effect.
5800 ConfigureEncoder(video_encoder_config.Copy());
5801
5802 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5803 kMaxPayloadLength);
5804 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5805
5806 // The default bitrate limits for 360p should not be used.
5807 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005808 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5809 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005810 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5811 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005812 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5813 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5814 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5815 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5816 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5817 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005818 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005819 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005820
5821 video_stream_encoder_->Stop();
5822}
5823
5824TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5825 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5826 /*num_spatial_layers=*/1, /*screenshare=*/false);
5827
5828 // The default singlecast bitrate limits for 720p should not be used.
5829 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005830 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5831 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005832 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5833 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005834 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5835 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5836 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5837 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5838 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5839 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005840 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005841 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005842
5843 video_stream_encoder_->Stop();
5844}
5845
5846TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005847 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5848 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5849 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5850 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5851 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5852 fake_encoder_.SetResolutionBitrateLimits(
5853 {kEncoderLimits180p, kEncoderLimits720p});
5854
5855 VideoEncoderConfig video_encoder_config;
5856 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5857 &video_encoder_config);
5858 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5859 vp9_settings.numberOfSpatialLayers = 3;
5860 // Since only one layer is active - automatic resize should be enabled.
5861 vp9_settings.automaticResizeOn = true;
5862 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005863 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005864 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005865 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005866 video_encoder_config.content_type =
5867 VideoEncoderConfig::ContentType::kRealtimeVideo;
5868 // Simulcast layers are used to indicate which spatial layers are active.
5869 video_encoder_config.simulcast_layers.resize(3);
5870 video_encoder_config.simulcast_layers[0].active = true;
5871 video_encoder_config.simulcast_layers[1].active = false;
5872 video_encoder_config.simulcast_layers[2].active = false;
5873
5874 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5875 kMaxPayloadLength);
5876 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5877
5878 // Limits not applied on lowest stream, limits for 180p should not be used.
5879 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5880 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005881 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5882 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5883 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5884 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5885 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5886 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005887 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005888 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005889 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005890 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005891
5892 video_stream_encoder_->Stop();
5893}
5894
5895TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005896 InitialFrameDropActivatesWhenResolutionIncreases) {
5897 const int kWidth = 640;
5898 const int kHeight = 360;
5899
5900 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005901 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005902 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5903 // Frame should not be dropped.
5904 WaitForEncodedFrame(1);
5905
5906 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005907 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005908 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5909 // Frame should not be dropped, bitrate not too low for frame.
5910 WaitForEncodedFrame(2);
5911
5912 // Incoming resolution increases.
5913 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5914 // Expect to drop this frame, bitrate too low for frame.
5915 ExpectDroppedFrame();
5916
5917 // Expect the sink_wants to specify a scaled frame.
5918 EXPECT_TRUE_WAIT(
5919 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5920 video_stream_encoder_->Stop();
5921}
5922
5923TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5924 const int kWidth = 640;
5925 const int kHeight = 360;
5926 // So that quality scaling doesn't happen by itself.
5927 fake_encoder_.SetQp(kQpHigh);
5928
5929 AdaptingFrameForwarder source(&time_controller_);
5930 source.set_adaptation_enabled(true);
5931 video_stream_encoder_->SetSource(
5932 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5933
5934 int timestamp = 1;
5935
5936 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005937 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005938 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5939 WaitForEncodedFrame(timestamp);
5940 timestamp += 9000;
5941 // Long pause to disable all first BWE drop logic.
5942 AdvanceTime(TimeDelta::Millis(1000));
5943
5944 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005945 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005946 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5947 // Not dropped frame, as initial frame drop is disabled by now.
5948 WaitForEncodedFrame(timestamp);
5949 timestamp += 9000;
5950 AdvanceTime(TimeDelta::Millis(100));
5951
5952 // Quality adaptation down.
5953 video_stream_encoder_->TriggerQualityLow();
5954
5955 // Adaptation has an effect.
5956 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5957 5000);
5958
5959 // Frame isn't dropped as initial frame dropper is disabled.
5960 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5961 WaitForEncodedFrame(timestamp);
5962 timestamp += 9000;
5963 AdvanceTime(TimeDelta::Millis(100));
5964
5965 // Quality adaptation up.
5966 video_stream_encoder_->TriggerQualityHigh();
5967
5968 // Adaptation has an effect.
5969 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5970 5000);
5971
5972 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5973 // Frame should not be dropped, as initial framedropper is off.
5974 WaitForEncodedFrame(timestamp);
5975
5976 video_stream_encoder_->Stop();
5977}
5978
Åsa Persson7f354f82021-02-04 15:52:15 +01005979TEST_F(VideoStreamEncoderTest,
5980 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5981 const int kMinStartBps360p = 222000;
5982 fake_encoder_.SetResolutionBitrateLimits(
5983 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5984 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5985 800000)});
5986
5987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5988 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5989 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5990 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5991 0, 0, 0);
5992 // Frame should not be dropped, bitrate not too low for frame.
5993 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5994 WaitForEncodedFrame(1);
5995
5996 // Incoming resolution increases, initial frame drop activates.
5997 // Frame should be dropped, link allocation too low for frame.
5998 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5999 ExpectDroppedFrame();
6000
6001 // Expect sink_wants to specify a scaled frame.
6002 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6003 5000);
6004 video_stream_encoder_->Stop();
6005}
6006
6007TEST_F(VideoStreamEncoderTest,
6008 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6009 const int kMinStartBps360p = 222000;
6010 fake_encoder_.SetResolutionBitrateLimits(
6011 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6012 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6013 800000)});
6014
6015 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6016 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6017 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6018 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6019 0, 0, 0);
6020 // Frame should not be dropped, bitrate not too low for frame.
6021 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6022 WaitForEncodedFrame(1);
6023
6024 // Incoming resolution increases, initial frame drop activates.
6025 // Frame should be dropped, link allocation not too low for frame.
6026 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6027 WaitForEncodedFrame(2);
6028
6029 video_stream_encoder_->Stop();
6030}
6031
Åsa Perssone644a032019-11-08 15:56:00 +01006032TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6033 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006034 "WebRTC-Video-QualityRampupSettings/"
6035 "min_pixels:921600,min_duration_ms:2000/");
6036
6037 const int kWidth = 1280;
6038 const int kHeight = 720;
6039 const int kFps = 10;
6040 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006041
6042 // Reset encoder for field trials to take effect.
6043 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006044 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006045 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006046 ConfigureEncoder(std::move(config));
6047 fake_encoder_.SetQp(kQpLow);
6048
6049 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006050 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006051 source.set_adaptation_enabled(true);
6052 video_stream_encoder_->SetSource(&source,
6053 DegradationPreference::MAINTAIN_FRAMERATE);
6054
6055 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006056 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006058 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006059
6060 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006061 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006062 int64_t timestamp_ms = kFrameIntervalMs;
6063 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6064 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006065 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6066 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006067
6068 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006069 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6070 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006071
Artem Titovab30d722021-07-27 16:22:11 +02006072 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006073 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006074 for (size_t i = 1; i <= 10; i++) {
6075 timestamp_ms += kFrameIntervalMs;
6076 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6077 WaitForEncodedFrame(timestamp_ms);
6078 }
Åsa Persson06defc42021-09-10 15:28:48 +02006079
6080 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6081 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6082 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6083 timestamp_ms += kFrameIntervalMs;
6084 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6085 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006086 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6087 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6088
Åsa Persson06defc42021-09-10 15:28:48 +02006089 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006090 timestamp_ms += kFrameIntervalMs;
6091 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6092 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006093 // The ramp-up code involves the adaptation queue, give it time to execute.
6094 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006095 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006096 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006097
6098 // Frame should not be adapted.
6099 timestamp_ms += kFrameIntervalMs;
6100 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6101 WaitForEncodedFrame(kWidth, kHeight);
6102 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6103
6104 video_stream_encoder_->Stop();
6105}
6106
mflodmancc3d4422017-08-03 08:27:51 -07006107TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006108 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006109 webrtc::test::ScopedFieldTrials field_trials(
6110 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006111 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006112 source.set_adaptation_enabled(true);
6113 video_stream_encoder_->SetSource(&source,
6114 DegradationPreference::MAINTAIN_FRAMERATE);
6115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006116 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006117 fake_encoder_.SetQp(kQpHigh + 1);
6118 const int kWidth = 1280;
6119 const int kHeight = 720;
6120 const int64_t kFrameIntervalMs = 100;
6121 int64_t timestamp_ms = kFrameIntervalMs;
6122 for (size_t i = 1; i <= 100; i++) {
6123 timestamp_ms += kFrameIntervalMs;
6124 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6125 WaitForEncodedFrame(timestamp_ms);
6126 }
6127 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6128 // for the first time.
6129 // TODO(eshr): We should avoid these waits by using threads with simulated
6130 // time.
6131 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6132 2000 * 2.5 * 2);
6133 timestamp_ms += kFrameIntervalMs;
6134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6135 WaitForEncodedFrame(timestamp_ms);
6136 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6137 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6138 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6139
6140 // Disable Quality scaling by turning off scaler on the encoder and
6141 // reconfiguring.
6142 fake_encoder_.SetQualityScaling(false);
6143 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6144 kMaxPayloadLength);
6145 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006146 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006147 // Since we turned off the quality scaler, the adaptations made by it are
6148 // removed.
6149 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6150 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6151
6152 video_stream_encoder_->Stop();
6153}
6154
6155TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006156 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6157 const int kTooSmallWidth = 10;
6158 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006160 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006161
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006162 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006163 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006165 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006166 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006167 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6168
6169 // Trigger adapt down, too small frame, expect no change.
6170 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006171 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006172 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006173 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006174 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6175 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6176
mflodmancc3d4422017-08-03 08:27:51 -07006177 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006178}
6179
mflodmancc3d4422017-08-03 08:27:51 -07006180TEST_F(VideoStreamEncoderTest,
6181 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006182 const int kTooSmallWidth = 10;
6183 const int kTooSmallHeight = 10;
6184 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006185 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006186 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006187
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006188 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006189 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006190 video_stream_encoder_->SetSource(&source,
6191 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006192 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6195
6196 // Trigger adapt down, expect limited framerate.
6197 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006198 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006199 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006200 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006201 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6202 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6203 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6204
6205 // Trigger adapt down, too small frame, expect no change.
6206 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006207 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006208 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006209 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006210 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6211 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6212 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6213
mflodmancc3d4422017-08-03 08:27:51 -07006214 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006215}
6216
mflodmancc3d4422017-08-03 08:27:51 -07006217TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006218 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006220 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006221 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006222 const int kFrameWidth = 1280;
6223 const int kFrameHeight = 720;
6224 video_source_.IncomingCapturedFrame(
6225 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006226 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006227 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006228}
6229
sprangb1ca0732017-02-01 08:38:12 -08006230// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006231TEST_F(VideoStreamEncoderTest,
6232 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006233 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006234 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006235
6236 const int kFrameWidth = 1280;
6237 const int kFrameHeight = 720;
6238 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006239 // requested by
6240 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006241 video_source_.set_adaptation_enabled(true);
6242
6243 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006244 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006245 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006246
6247 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006248 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006249 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006250 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006251 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006252
asaperssonfab67072017-04-04 05:51:49 -07006253 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006254 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006255 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006256 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006257 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006258
mflodmancc3d4422017-08-03 08:27:51 -07006259 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006260}
sprangfe627f32017-03-29 08:24:59 -07006261
mflodmancc3d4422017-08-03 08:27:51 -07006262TEST_F(VideoStreamEncoderTest,
6263 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006264 const int kFrameWidth = 1280;
6265 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006266
Henrik Boström381d1092020-05-12 18:49:07 +02006267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006268 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006269 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006270 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006271 video_source_.set_adaptation_enabled(true);
6272
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006273 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006274
6275 video_source_.IncomingCapturedFrame(
6276 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006277 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006278
6279 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006280 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006281
6282 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006283 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006284 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006285 video_source_.IncomingCapturedFrame(
6286 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006287 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006288 }
6289
6290 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006292 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006293 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006294 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006295 video_source_.IncomingCapturedFrame(
6296 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006297 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006298 ++num_frames_dropped;
6299 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006300 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006301 }
6302 }
6303
sprang4847ae62017-06-27 07:06:52 -07006304 // Add some slack to account for frames dropped by the frame dropper.
6305 const int kErrorMargin = 1;
6306 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006307 kErrorMargin);
6308
6309 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006310 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006311 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006312 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006313 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006314 video_source_.IncomingCapturedFrame(
6315 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006316 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006317 ++num_frames_dropped;
6318 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006319 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006320 }
6321 }
sprang4847ae62017-06-27 07:06:52 -07006322 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006323 kErrorMargin);
6324
6325 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006326 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006327 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006328 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006329 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006330 video_source_.IncomingCapturedFrame(
6331 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006332 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006333 ++num_frames_dropped;
6334 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006335 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006336 }
6337 }
sprang4847ae62017-06-27 07:06:52 -07006338 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006339 kErrorMargin);
6340
6341 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006342 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006343 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006344 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006345 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006346 video_source_.IncomingCapturedFrame(
6347 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006348 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006349 ++num_frames_dropped;
6350 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006351 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006352 }
6353 }
6354 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6355
mflodmancc3d4422017-08-03 08:27:51 -07006356 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006357}
6358
mflodmancc3d4422017-08-03 08:27:51 -07006359TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006360 const int kFramerateFps = 5;
6361 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006362 const int kFrameWidth = 1280;
6363 const int kFrameHeight = 720;
6364
sprang4847ae62017-06-27 07:06:52 -07006365 // Reconfigure encoder with two temporal layers and screensharing, which will
6366 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006367 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006368
Henrik Boström381d1092020-05-12 18:49:07 +02006369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006370 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006371 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006372 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006373 video_source_.set_adaptation_enabled(true);
6374
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006375 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006376
6377 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006378 rtc::VideoSinkWants last_wants;
6379 do {
6380 last_wants = video_source_.sink_wants();
6381
sprangc5d62e22017-04-02 23:53:04 -07006382 // Insert frames to get a new fps estimate...
6383 for (int j = 0; j < kFramerateFps; ++j) {
6384 video_source_.IncomingCapturedFrame(
6385 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006386 if (video_source_.last_sent_width()) {
6387 sink_.WaitForEncodedFrame(timestamp_ms);
6388 }
sprangc5d62e22017-04-02 23:53:04 -07006389 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006390 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006391 }
6392 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006393 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006394 } while (video_source_.sink_wants().max_framerate_fps <
6395 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006396
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006397 EXPECT_THAT(video_source_.sink_wants(),
6398 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006399
mflodmancc3d4422017-08-03 08:27:51 -07006400 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006401}
asaperssonf7e294d2017-06-13 23:25:22 -07006402
mflodmancc3d4422017-08-03 08:27:51 -07006403TEST_F(VideoStreamEncoderTest,
6404 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006405 const int kWidth = 1280;
6406 const int kHeight = 720;
6407 const int64_t kFrameIntervalMs = 150;
6408 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006409 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006410 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006411
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006412 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006413 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006414 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006415 video_stream_encoder_->SetSource(&source,
6416 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006417 timestamp_ms += kFrameIntervalMs;
6418 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006419 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006420 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6422 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6423 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6424
6425 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006426 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006427 timestamp_ms += kFrameIntervalMs;
6428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006429 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006430 EXPECT_THAT(source.sink_wants(),
6431 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006432 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6434 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6435
6436 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006437 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006438 timestamp_ms += kFrameIntervalMs;
6439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006440 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006441 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006442 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6444 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6445
6446 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006447 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006448 timestamp_ms += kFrameIntervalMs;
6449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006450 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006451 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006452 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6454 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6455
6456 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006457 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006458 timestamp_ms += kFrameIntervalMs;
6459 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006460 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006461 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006462 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6463 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6464 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6465
6466 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006467 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006468 timestamp_ms += kFrameIntervalMs;
6469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006470 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006471 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6473 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6474 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6475
6476 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006477 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006478 timestamp_ms += kFrameIntervalMs;
6479 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006480 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006481 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006482 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6484 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6485
6486 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006487 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006488 timestamp_ms += kFrameIntervalMs;
6489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006490 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006491 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006492 rtc::VideoSinkWants last_wants = source.sink_wants();
6493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6495 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6496
6497 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006498 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006499 timestamp_ms += kFrameIntervalMs;
6500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006501 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006502 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006503 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6505 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6506
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006507 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006508 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006509 timestamp_ms += kFrameIntervalMs;
6510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006511 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006512 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6515 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6516
6517 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006518 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006519 timestamp_ms += kFrameIntervalMs;
6520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006521 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006522 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6525 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6526
6527 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006528 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006529 timestamp_ms += kFrameIntervalMs;
6530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006531 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006532 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6535 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6536
6537 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006538 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006539 timestamp_ms += kFrameIntervalMs;
6540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006541 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006542 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6545 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6546
6547 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006548 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006549 timestamp_ms += kFrameIntervalMs;
6550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006551 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006552 EXPECT_THAT(source.sink_wants(), FpsMax());
6553 EXPECT_EQ(source.sink_wants().max_pixel_count,
6554 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6556 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6557 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6558
6559 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006560 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006561 timestamp_ms += kFrameIntervalMs;
6562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6567 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6568
Åsa Persson30ab0152019-08-27 12:22:33 +02006569 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006570 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006571 timestamp_ms += kFrameIntervalMs;
6572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006573 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006574 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006575 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006576 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6577 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6578 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6579
6580 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006581 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006582 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006583 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6584
mflodmancc3d4422017-08-03 08:27:51 -07006585 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006586}
6587
mflodmancc3d4422017-08-03 08:27:51 -07006588TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006589 const int kWidth = 1280;
6590 const int kHeight = 720;
6591 const int64_t kFrameIntervalMs = 150;
6592 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006594 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006595
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006596 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006597 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006598 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006599 video_stream_encoder_->SetSource(&source,
6600 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006601 timestamp_ms += kFrameIntervalMs;
6602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006603 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006604 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006605 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6609 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6610 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611
6612 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006613 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006614 timestamp_ms += kFrameIntervalMs;
6615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006616 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006617 EXPECT_THAT(source.sink_wants(),
6618 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6620 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6621 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6622 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6623 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6624 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6625
6626 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006627 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006628 timestamp_ms += kFrameIntervalMs;
6629 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006630 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006631 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006632 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6634 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6635 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6636 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6637 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6638
6639 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006640 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006641 timestamp_ms += kFrameIntervalMs;
6642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006643 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006644 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6647 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6648 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6649 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6650 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6651
Evan Shrubsole64469032020-06-11 10:45:29 +02006652 // Trigger cpu adapt up, expect no change since QP is most limited.
6653 {
6654 // Store current sink wants since we expect no change and if there is no
6655 // change then last_wants() is not updated.
6656 auto previous_sink_wants = source.sink_wants();
6657 video_stream_encoder_->TriggerCpuUnderuse();
6658 timestamp_ms += kFrameIntervalMs;
6659 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6660 WaitForEncodedFrame(timestamp_ms);
6661 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6662 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6663 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6664 }
6665
6666 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6667 video_stream_encoder_->TriggerQualityHigh();
6668 timestamp_ms += kFrameIntervalMs;
6669 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6670 WaitForEncodedFrame(timestamp_ms);
6671 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6672 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6674 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6675 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6676 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6677 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6678
6679 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6680 // expect increased resolution (960x540@30fps).
6681 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006682 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006683 timestamp_ms += kFrameIntervalMs;
6684 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006685 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006687 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6688 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6689 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6690 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6691 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006692 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006693
Evan Shrubsole64469032020-06-11 10:45:29 +02006694 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6695 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006696 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006697 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006698 timestamp_ms += kFrameIntervalMs;
6699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006700 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006701 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006702 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006703 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6704 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6705 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6706 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6707 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006708 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006709
6710 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006711 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006712 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006713 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006714 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006715
mflodmancc3d4422017-08-03 08:27:51 -07006716 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006717}
6718
mflodmancc3d4422017-08-03 08:27:51 -07006719TEST_F(VideoStreamEncoderTest,
6720 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006721 const int kWidth = 640;
6722 const int kHeight = 360;
6723 const int kFpsLimit = 15;
6724 const int64_t kFrameIntervalMs = 150;
6725 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006727 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006728
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006729 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006730 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006731 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006732 video_stream_encoder_->SetSource(&source,
6733 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006734 timestamp_ms += kFrameIntervalMs;
6735 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006736 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006737 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006738 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6739 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6740 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6741 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6742 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6743 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6744
6745 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006746 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006747 timestamp_ms += kFrameIntervalMs;
6748 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006749 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006750 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006751 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6752 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6753 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6754 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6756 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6757
6758 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006759 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006760 timestamp_ms += kFrameIntervalMs;
6761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006762 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006763 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006764 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006765 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006766 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6767 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6768 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6769 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6770
Evan Shrubsole64469032020-06-11 10:45:29 +02006771 // Trigger cpu adapt up, expect no change because quality is most limited.
6772 {
6773 auto previous_sink_wants = source.sink_wants();
6774 // Store current sink wants since we expect no change ind if there is no
6775 // change then last__wants() is not updated.
6776 video_stream_encoder_->TriggerCpuUnderuse();
6777 timestamp_ms += kFrameIntervalMs;
6778 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6779 WaitForEncodedFrame(timestamp_ms);
6780 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6781 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6782 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6783 }
6784
6785 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6786 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006787 timestamp_ms += kFrameIntervalMs;
6788 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006789 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006790 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006791 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6792 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6793 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006794 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6795 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6796 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006797
Evan Shrubsole64469032020-06-11 10:45:29 +02006798 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006799 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006800 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006801 timestamp_ms += kFrameIntervalMs;
6802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006803 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006805 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6807 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6809 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006810 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006811
6812 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006813 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006815 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006816 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006817
mflodmancc3d4422017-08-03 08:27:51 -07006818 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006819}
6820
mflodmancc3d4422017-08-03 08:27:51 -07006821TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006822 const int kFrameWidth = 1920;
6823 const int kFrameHeight = 1080;
6824 // 3/4 of 1920.
6825 const int kAdaptedFrameWidth = 1440;
6826 // 3/4 of 1080 rounded down to multiple of 4.
6827 const int kAdaptedFrameHeight = 808;
6828 const int kFramerate = 24;
6829
Henrik Boström381d1092020-05-12 18:49:07 +02006830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006831 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006832 // Trigger reconfigure encoder (without resetting the entire instance).
6833 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006834 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6835 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006836 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006837 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006838 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006839 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006840 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006841 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006842
6843 video_source_.set_adaptation_enabled(true);
6844
6845 video_source_.IncomingCapturedFrame(
6846 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006847 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006848
6849 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006850 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006851 video_source_.IncomingCapturedFrame(
6852 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006853 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006854
mflodmancc3d4422017-08-03 08:27:51 -07006855 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006856}
6857
mflodmancc3d4422017-08-03 08:27:51 -07006858TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006859 const int kFrameWidth = 1280;
6860 const int kFrameHeight = 720;
6861 const int kLowFps = 2;
6862 const int kHighFps = 30;
6863
Henrik Boström381d1092020-05-12 18:49:07 +02006864 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006865 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006866
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006867 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006868 max_framerate_ = kLowFps;
6869
6870 // Insert 2 seconds of 2fps video.
6871 for (int i = 0; i < kLowFps * 2; ++i) {
6872 video_source_.IncomingCapturedFrame(
6873 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6874 WaitForEncodedFrame(timestamp_ms);
6875 timestamp_ms += 1000 / kLowFps;
6876 }
6877
6878 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006879 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006880 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006881 video_source_.IncomingCapturedFrame(
6882 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6883 WaitForEncodedFrame(timestamp_ms);
6884 timestamp_ms += 1000 / kLowFps;
6885
6886 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6887
6888 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006889 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006890 const int kFrameIntervalMs = 1000 / kHighFps;
6891 max_framerate_ = kHighFps;
6892 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6893 video_source_.IncomingCapturedFrame(
6894 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6895 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6896 // be dropped if the encoder hans't been updated with the new higher target
6897 // framerate yet, causing it to overshoot the target bitrate and then
6898 // suffering the wrath of the media optimizer.
6899 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6900 timestamp_ms += kFrameIntervalMs;
6901 }
6902
6903 // Don expect correct measurement just yet, but it should be higher than
6904 // before.
6905 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6906
mflodmancc3d4422017-08-03 08:27:51 -07006907 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006908}
6909
mflodmancc3d4422017-08-03 08:27:51 -07006910TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006911 const int kFrameWidth = 1280;
6912 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006913 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006914 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006915 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006916
Henrik Boström381d1092020-05-12 18:49:07 +02006917 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006918 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006919 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006920
6921 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006922 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006923 video_source_.IncomingCapturedFrame(
6924 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6925 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006926 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006927
6928 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006930 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006931
6932 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006933 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006934 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006935
Per Kjellanderdcef6412020-10-07 15:09:05 +02006936 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006937 video_source_.IncomingCapturedFrame(
6938 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6939 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006940 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006941
mflodmancc3d4422017-08-03 08:27:51 -07006942 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006943}
ilnik6b826ef2017-06-16 06:53:48 -07006944
Niels Möller4db138e2018-04-19 09:04:13 +02006945TEST_F(VideoStreamEncoderTest,
6946 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6947 const int kFrameWidth = 1280;
6948 const int kFrameHeight = 720;
6949 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006950 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006951 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006952 video_source_.IncomingCapturedFrame(
6953 CreateFrame(1, kFrameWidth, kFrameHeight));
6954 WaitForEncodedFrame(1);
6955 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6956 .low_encode_usage_threshold_percent,
6957 default_options.low_encode_usage_threshold_percent);
6958 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6959 .high_encode_usage_threshold_percent,
6960 default_options.high_encode_usage_threshold_percent);
6961 video_stream_encoder_->Stop();
6962}
6963
6964TEST_F(VideoStreamEncoderTest,
6965 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6966 const int kFrameWidth = 1280;
6967 const int kFrameHeight = 720;
6968 CpuOveruseOptions hardware_options;
6969 hardware_options.low_encode_usage_threshold_percent = 150;
6970 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006971 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006972
Henrik Boström381d1092020-05-12 18:49:07 +02006973 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006974 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006975 video_source_.IncomingCapturedFrame(
6976 CreateFrame(1, kFrameWidth, kFrameHeight));
6977 WaitForEncodedFrame(1);
6978 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6979 .low_encode_usage_threshold_percent,
6980 hardware_options.low_encode_usage_threshold_percent);
6981 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6982 .high_encode_usage_threshold_percent,
6983 hardware_options.high_encode_usage_threshold_percent);
6984 video_stream_encoder_->Stop();
6985}
6986
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006987TEST_F(VideoStreamEncoderTest,
6988 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6989 const int kFrameWidth = 1280;
6990 const int kFrameHeight = 720;
6991
6992 const CpuOveruseOptions default_options;
6993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(1, kFrameWidth, kFrameHeight));
6997 WaitForEncodedFrame(1);
6998 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6999 .low_encode_usage_threshold_percent,
7000 default_options.low_encode_usage_threshold_percent);
7001 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7002 .high_encode_usage_threshold_percent,
7003 default_options.high_encode_usage_threshold_percent);
7004
7005 CpuOveruseOptions hardware_options;
7006 hardware_options.low_encode_usage_threshold_percent = 150;
7007 hardware_options.high_encode_usage_threshold_percent = 200;
7008 fake_encoder_.SetIsHardwareAccelerated(true);
7009
7010 video_source_.IncomingCapturedFrame(
7011 CreateFrame(2, kFrameWidth, kFrameHeight));
7012 WaitForEncodedFrame(2);
7013
7014 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7015 .low_encode_usage_threshold_percent,
7016 hardware_options.low_encode_usage_threshold_percent);
7017 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7018 .high_encode_usage_threshold_percent,
7019 hardware_options.high_encode_usage_threshold_percent);
7020
7021 video_stream_encoder_->Stop();
7022}
7023
Niels Möller6bb5ab92019-01-11 11:11:10 +01007024TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7025 const int kFrameWidth = 320;
7026 const int kFrameHeight = 240;
7027 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007028 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007029 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7030
Henrik Boström381d1092020-05-12 18:49:07 +02007031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007033
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007034 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007035 max_framerate_ = kFps;
7036
7037 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7038 fake_encoder_.SimulateOvershoot(1.0);
7039 int num_dropped = 0;
7040 for (int i = 0; i < kNumFramesInRun; ++i) {
7041 video_source_.IncomingCapturedFrame(
7042 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7043 // Wait up to two frame durations for a frame to arrive.
7044 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7045 ++num_dropped;
7046 }
7047 timestamp_ms += 1000 / kFps;
7048 }
7049
Erik Språnga8d48ab2019-02-08 14:17:40 +01007050 // Framerate should be measured to be near the expected target rate.
7051 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7052
7053 // Frame drops should be within 5% of expected 0%.
7054 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007055
7056 // Make encoder produce frames at double the expected bitrate during 3 seconds
7057 // of video, verify number of drops. Rate needs to be slightly changed in
7058 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007059 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007060 const RateControlSettings trials =
7061 RateControlSettings::ParseFromFieldTrials();
7062 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007063 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007064 // frame dropping since the adjuter will try to just lower the target
7065 // bitrate rather than drop frames. If network headroom can be used, it
7066 // doesn't push back as hard so we don't need quite as much overshoot.
7067 // These numbers are unfortunately a bit magical but there's not trivial
7068 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007069 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007070 }
7071 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007073 kTargetBitrate + DataRate::KilobitsPerSec(1),
7074 kTargetBitrate + DataRate::KilobitsPerSec(1),
7075 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007076 num_dropped = 0;
7077 for (int i = 0; i < kNumFramesInRun; ++i) {
7078 video_source_.IncomingCapturedFrame(
7079 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7080 // Wait up to two frame durations for a frame to arrive.
7081 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7082 ++num_dropped;
7083 }
7084 timestamp_ms += 1000 / kFps;
7085 }
7086
Henrik Boström381d1092020-05-12 18:49:07 +02007087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007088 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007089
7090 // Target framerate should be still be near the expected target, despite
7091 // the frame drops.
7092 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7093
7094 // Frame drops should be within 5% of expected 50%.
7095 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007096
7097 video_stream_encoder_->Stop();
7098}
7099
7100TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7101 const int kFrameWidth = 320;
7102 const int kFrameHeight = 240;
7103 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007104 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007105
7106 ASSERT_GT(max_framerate_, kActualInputFps);
7107
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007108 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007109 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007112
7113 // Insert 3 seconds of video, with an input fps lower than configured max.
7114 for (int i = 0; i < kActualInputFps * 3; ++i) {
7115 video_source_.IncomingCapturedFrame(
7116 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7117 // Wait up to two frame durations for a frame to arrive.
7118 WaitForEncodedFrame(timestamp_ms);
7119 timestamp_ms += 1000 / kActualInputFps;
7120 }
7121
7122 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7123
7124 video_stream_encoder_->Stop();
7125}
7126
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007127TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007128 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007129 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007130 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007131
7132 fake_encoder_.BlockNextEncode();
7133 video_source_.IncomingCapturedFrame(
7134 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7135 WaitForEncodedFrame(1);
7136 // On the very first frame full update should be forced.
7137 rect = fake_encoder_.GetLastUpdateRect();
7138 EXPECT_EQ(rect.offset_x, 0);
7139 EXPECT_EQ(rect.offset_y, 0);
7140 EXPECT_EQ(rect.height, codec_height_);
7141 EXPECT_EQ(rect.width, codec_width_);
7142 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7143 // call to ContinueEncode.
7144 video_source_.IncomingCapturedFrame(
7145 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7146 ExpectDroppedFrame();
7147 video_source_.IncomingCapturedFrame(
7148 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7149 ExpectDroppedFrame();
7150 fake_encoder_.ContinueEncode();
7151 WaitForEncodedFrame(3);
7152 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7153 rect = fake_encoder_.GetLastUpdateRect();
7154 EXPECT_EQ(rect.offset_x, 1);
7155 EXPECT_EQ(rect.offset_y, 0);
7156 EXPECT_EQ(rect.width, 10);
7157 EXPECT_EQ(rect.height, 1);
7158
7159 video_source_.IncomingCapturedFrame(
7160 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7161 WaitForEncodedFrame(4);
7162 // Previous frame was encoded, so no accumulation should happen.
7163 rect = fake_encoder_.GetLastUpdateRect();
7164 EXPECT_EQ(rect.offset_x, 0);
7165 EXPECT_EQ(rect.offset_y, 0);
7166 EXPECT_EQ(rect.width, 1);
7167 EXPECT_EQ(rect.height, 1);
7168
7169 video_stream_encoder_->Stop();
7170}
7171
Erik Språngd7329ca2019-02-21 21:19:53 +01007172TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007174 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007175
7176 // First frame is always keyframe.
7177 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7178 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007179 EXPECT_THAT(
7180 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007181 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007182
7183 // Insert delta frame.
7184 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7185 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007186 EXPECT_THAT(
7187 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007188 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007189
7190 // Request next frame be a key-frame.
7191 video_stream_encoder_->SendKeyFrame();
7192 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7193 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007194 EXPECT_THAT(
7195 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007196 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007197
7198 video_stream_encoder_->Stop();
7199}
7200
7201TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7202 // Setup simulcast with three streams.
7203 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007205 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7206 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007207 // Wait for all three layers before triggering event.
7208 sink_.SetNumExpectedLayers(3);
7209
7210 // First frame is always keyframe.
7211 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7212 WaitForEncodedFrame(1);
7213 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007214 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7215 VideoFrameType::kVideoFrameKey,
7216 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007217
7218 // Insert delta frame.
7219 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7220 WaitForEncodedFrame(2);
7221 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007222 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7223 VideoFrameType::kVideoFrameDelta,
7224 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007225
7226 // Request next frame be a key-frame.
7227 // Only first stream is configured to produce key-frame.
7228 video_stream_encoder_->SendKeyFrame();
7229 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7230 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007231
7232 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7233 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007234 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007235 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007236 VideoFrameType::kVideoFrameKey,
7237 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007238
7239 video_stream_encoder_->Stop();
7240}
7241
7242TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7243 // Configure internal source factory and setup test again.
7244 encoder_factory_.SetHasInternalSource(true);
7245 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007246 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007247 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007248
7249 // Call encoder directly, simulating internal source where encoded frame
7250 // callback in VideoStreamEncoder is called despite no OnFrame().
7251 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7252 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007253 EXPECT_THAT(
7254 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007255 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007256
Niels Möller8f7ce222019-03-21 15:43:58 +01007257 const std::vector<VideoFrameType> kDeltaFrame = {
7258 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007259 // Need to set timestamp manually since manually for injected frame.
7260 VideoFrame frame = CreateFrame(101, nullptr);
7261 frame.set_timestamp(101);
7262 fake_encoder_.InjectFrame(frame, false);
7263 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007264 EXPECT_THAT(
7265 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007266 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007267
7268 // Request key-frame. The forces a dummy frame down into the encoder.
7269 fake_encoder_.ExpectNullFrame();
7270 video_stream_encoder_->SendKeyFrame();
7271 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007272 EXPECT_THAT(
7273 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007274 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007275
7276 video_stream_encoder_->Stop();
7277}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007278
7279TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7280 // Configure internal source factory and setup test again.
7281 encoder_factory_.SetHasInternalSource(true);
7282 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007284 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007285
7286 int64_t timestamp = 1;
7287 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007288 image.capture_time_ms_ = ++timestamp;
7289 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7290 const int64_t kEncodeFinishDelayMs = 10;
7291 image.timing_.encode_start_ms = timestamp;
7292 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007293 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007294 // Wait for frame without incrementing clock.
7295 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7296 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7297 // capture timestamp should be kEncodeFinishDelayMs in the past.
7298 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007299 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007300
7301 video_stream_encoder_->Stop();
7302}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007303
7304TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007305 // SPS contains VUI with restrictions on the maximum number of reordered
7306 // pictures, there is no need to rewrite the bitstream to enable faster
7307 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007308 ResetEncoder("H264", 1, 1, 1, false);
7309
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007311 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007312 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007313
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007314 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007315 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007316
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007317 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7318 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007319
7320 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007321 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007322
7323 video_stream_encoder_->Stop();
7324}
7325
7326TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007327 // SPS does not contain VUI, the bitstream is will be rewritten with added
7328 // VUI with restrictions on the maximum number of reordered pictures to
7329 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007330 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7331 0x00, 0x00, 0x03, 0x03, 0xF4,
7332 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007333 ResetEncoder("H264", 1, 1, 1, false);
7334
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007336 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007337 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007338
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007339 fake_encoder_.SetEncodedImageData(
7340 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007341
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007342 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7343 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007344
7345 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007346 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007347
7348 video_stream_encoder_->Stop();
7349}
7350
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007351TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7352 const int kFrameWidth = 1280;
7353 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007354 const DataRate kTargetBitrate =
7355 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007356
Henrik Boström381d1092020-05-12 18:49:07 +02007357 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007358 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7360
7361 // Insert a first video frame. It should be dropped because of downscale in
7362 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007363 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007364 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7365 frame.set_rotation(kVideoRotation_270);
7366 video_source_.IncomingCapturedFrame(frame);
7367
7368 ExpectDroppedFrame();
7369
7370 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007371 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007372 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7373 frame.set_rotation(kVideoRotation_90);
7374 video_source_.IncomingCapturedFrame(frame);
7375
7376 WaitForEncodedFrame(timestamp_ms);
7377 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7378
7379 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007380 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007381 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7382 frame.set_rotation(kVideoRotation_180);
7383 video_source_.IncomingCapturedFrame(frame);
7384
7385 WaitForEncodedFrame(timestamp_ms);
7386 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7387
7388 video_stream_encoder_->Stop();
7389}
7390
Erik Språng5056af02019-09-02 15:53:11 +02007391TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7392 const int kFrameWidth = 320;
7393 const int kFrameHeight = 180;
7394
7395 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007397 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7398 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7399 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007400 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007401 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007402 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007403
7404 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007405 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007406 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7407 frame.set_rotation(kVideoRotation_270);
7408 video_source_.IncomingCapturedFrame(frame);
7409 WaitForEncodedFrame(timestamp_ms);
7410
7411 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007412 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007413 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7414 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007416 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007417 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007418 /*link_allocation=*/target_rate,
7419 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007420 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007421 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007422 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7423
7424 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7425 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7426 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007427 DataRate allocation_sum =
7428 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007429 EXPECT_EQ(min_rate, allocation_sum);
7430 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7431
7432 video_stream_encoder_->Stop();
7433}
7434
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007435TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007436 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007437 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007438 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007439 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007440 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7441 WaitForEncodedFrame(1);
7442
7443 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7444 ASSERT_TRUE(prev_rate_settings.has_value());
7445 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7446 kDefaultFramerate);
7447
7448 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7449 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7450 timestamp_ms += 1000 / kDefaultFramerate;
7451 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7452 WaitForEncodedFrame(timestamp_ms);
7453 }
7454 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7455 kDefaultFramerate);
7456 // Capture larger frame to trigger a reconfigure.
7457 codec_height_ *= 2;
7458 codec_width_ *= 2;
7459 timestamp_ms += 1000 / kDefaultFramerate;
7460 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7461 WaitForEncodedFrame(timestamp_ms);
7462
7463 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7464 auto current_rate_settings =
7465 fake_encoder_.GetAndResetLastRateControlSettings();
7466 // Ensure we have actually reconfigured twice
7467 // The rate settings should have been set again even though
7468 // they haven't changed.
7469 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007470 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007471
7472 video_stream_encoder_->Stop();
7473}
7474
philipeld9cc8c02019-09-16 14:53:40 +02007475struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007476 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7477 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7478 MOCK_METHOD(void,
7479 RequestEncoderSwitch,
7480 (const webrtc::SdpVideoFormat& format),
7481 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007482};
7483
philipel9b058032020-02-10 11:30:00 +01007484TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7485 constexpr int kDontCare = 100;
7486 StrictMock<MockEncoderSelector> encoder_selector;
7487 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7488 &fake_encoder_, &encoder_selector);
7489 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7490
7491 // Reset encoder for new configuration to take effect.
7492 ConfigureEncoder(video_encoder_config_.Copy());
7493
7494 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7495
7496 video_source_.IncomingCapturedFrame(
7497 CreateFrame(kDontCare, kDontCare, kDontCare));
7498 video_stream_encoder_->Stop();
7499
7500 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7501 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007502 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7503 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007504 video_stream_encoder_.reset();
7505}
7506
7507TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7508 constexpr int kDontCare = 100;
7509
7510 NiceMock<MockEncoderSelector> encoder_selector;
7511 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7512 video_send_config_.encoder_settings.encoder_switch_request_callback =
7513 &switch_callback;
7514 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7515 &fake_encoder_, &encoder_selector);
7516 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7517
7518 // Reset encoder for new configuration to take effect.
7519 ConfigureEncoder(video_encoder_config_.Copy());
7520
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007521 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007522 .WillByDefault(Return(SdpVideoFormat("AV1")));
7523 EXPECT_CALL(switch_callback,
7524 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7525 Field(&SdpVideoFormat::name, "AV1"))));
7526
Henrik Boström381d1092020-05-12 18:49:07 +02007527 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007528 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7529 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7530 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007531 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007532 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007533 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007534 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007535
7536 video_stream_encoder_->Stop();
7537}
7538
7539TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7540 constexpr int kSufficientBitrateToNotDrop = 1000;
7541 constexpr int kDontCare = 100;
7542
7543 NiceMock<MockVideoEncoder> video_encoder;
7544 NiceMock<MockEncoderSelector> encoder_selector;
7545 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7546 video_send_config_.encoder_settings.encoder_switch_request_callback =
7547 &switch_callback;
7548 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7549 &video_encoder, &encoder_selector);
7550 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7551
7552 // Reset encoder for new configuration to take effect.
7553 ConfigureEncoder(video_encoder_config_.Copy());
7554
7555 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7556 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7557 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007559 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7560 /*stable_target_bitrate=*/
7561 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7562 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007563 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007564 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007565 /*cwnd_reduce_ratio=*/0);
7566
7567 ON_CALL(video_encoder, Encode(_, _))
7568 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7569 ON_CALL(encoder_selector, OnEncoderBroken())
7570 .WillByDefault(Return(SdpVideoFormat("AV2")));
7571
7572 rtc::Event encode_attempted;
7573 EXPECT_CALL(switch_callback,
7574 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7575 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7576 EXPECT_EQ(format.name, "AV2");
7577 encode_attempted.Set();
7578 });
7579
7580 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7581 encode_attempted.Wait(3000);
7582
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007583 AdvanceTime(TimeDelta::Millis(0));
7584
philipel9b058032020-02-10 11:30:00 +01007585 video_stream_encoder_->Stop();
7586
7587 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7588 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007589 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7590 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007591 video_stream_encoder_.reset();
7592}
7593
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007594TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007595 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007596 const int kFrameWidth = 320;
7597 const int kFrameHeight = 180;
7598
7599 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007600 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007602 /*target_bitrate=*/rate,
7603 /*stable_target_bitrate=*/rate,
7604 /*link_allocation=*/rate,
7605 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007606 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007607 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007608
7609 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007610 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007611 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7612 frame.set_rotation(kVideoRotation_270);
7613 video_source_.IncomingCapturedFrame(frame);
7614 WaitForEncodedFrame(timestamp_ms);
7615 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7616
7617 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007618 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007620 /*target_bitrate=*/new_stable_rate,
7621 /*stable_target_bitrate=*/new_stable_rate,
7622 /*link_allocation=*/rate,
7623 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007624 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007625 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007626 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7627 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7628 video_stream_encoder_->Stop();
7629}
7630
7631TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007632 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007633 const int kFrameWidth = 320;
7634 const int kFrameHeight = 180;
7635
7636 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007637 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007638 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007639 /*target_bitrate=*/rate,
7640 /*stable_target_bitrate=*/rate,
7641 /*link_allocation=*/rate,
7642 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007643 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007644 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007645
7646 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007647 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007648 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7649 frame.set_rotation(kVideoRotation_270);
7650 video_source_.IncomingCapturedFrame(frame);
7651 WaitForEncodedFrame(timestamp_ms);
7652 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7653
7654 // Set a higher target rate without changing the link_allocation. Should not
7655 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007656 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007658 /*target_bitrate=*/rate,
7659 /*stable_target_bitrate=*/new_stable_rate,
7660 /*link_allocation=*/rate,
7661 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007662 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007663 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007664 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7665 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7666 video_stream_encoder_->Stop();
7667}
7668
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007669TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7670 test::ScopedFieldTrials field_trials(
7671 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7672 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7673 const int kFramerateFps = 30;
7674 const int kWidth = 1920;
7675 const int kHeight = 1080;
7676 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7677 // Works on screenshare mode.
7678 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7679 // We rely on the automatic resolution adaptation, but we handle framerate
7680 // adaptation manually by mocking the stats proxy.
7681 video_source_.set_adaptation_enabled(true);
7682
7683 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007685 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007686 video_stream_encoder_->SetSource(&video_source_,
7687 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007688 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007689
7690 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7691 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7692
7693 // Pass enough frames with the full update to trigger animation detection.
7694 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007695 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007696 frame.set_ntp_time_ms(timestamp_ms);
7697 frame.set_timestamp_us(timestamp_ms * 1000);
7698 video_source_.IncomingCapturedFrame(frame);
7699 WaitForEncodedFrame(timestamp_ms);
7700 }
7701
7702 // Resolution should be limited.
7703 rtc::VideoSinkWants expected;
7704 expected.max_framerate_fps = kFramerateFps;
7705 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007706 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007707
7708 // Pass one frame with no known update.
7709 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007710 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007711 frame.set_ntp_time_ms(timestamp_ms);
7712 frame.set_timestamp_us(timestamp_ms * 1000);
7713 frame.clear_update_rect();
7714
7715 video_source_.IncomingCapturedFrame(frame);
7716 WaitForEncodedFrame(timestamp_ms);
7717
7718 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007719 EXPECT_THAT(video_source_.sink_wants(),
7720 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007721
7722 video_stream_encoder_->Stop();
7723}
7724
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007725TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7726 const int kWidth = 720; // 540p adapted down.
7727 const int kHeight = 405;
7728 const int kNumFrames = 3;
7729 // Works on screenshare mode.
7730 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7731 /*num_spatial_layers=*/2, /*screenshare=*/true);
7732
7733 video_source_.set_adaptation_enabled(true);
7734
7735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007736 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007737
7738 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7739
7740 // Pass enough frames with the full update to trigger animation detection.
7741 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007742 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007743 frame.set_ntp_time_ms(timestamp_ms);
7744 frame.set_timestamp_us(timestamp_ms * 1000);
7745 video_source_.IncomingCapturedFrame(frame);
7746 WaitForEncodedFrame(timestamp_ms);
7747 }
7748
7749 video_stream_encoder_->Stop();
7750}
7751
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007752TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7753 const float downscale_factors[] = {4.0, 2.0, 1.0};
7754 const int number_layers =
7755 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7756 VideoEncoderConfig config;
7757 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7758 for (int i = 0; i < number_layers; ++i) {
7759 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7760 config.simulcast_layers[i].active = true;
7761 }
7762 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007763 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007764 "VP8", /*max qp*/ 56, /*screencast*/ false,
7765 /*screenshare enabled*/ false);
7766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007767 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7768 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007769
7770 // First initialization.
7771 // Encoder should be initialized. Next frame should be key frame.
7772 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7773 sink_.SetNumExpectedLayers(number_layers);
7774 int64_t timestamp_ms = kFrameIntervalMs;
7775 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7776 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007777 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007778 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7779 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7780 VideoFrameType::kVideoFrameKey,
7781 VideoFrameType::kVideoFrameKey}));
7782
7783 // Disable top layer.
7784 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7785 config.simulcast_layers[number_layers - 1].active = false;
7786 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7787 sink_.SetNumExpectedLayers(number_layers - 1);
7788 timestamp_ms += kFrameIntervalMs;
7789 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7790 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007791 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007792 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7793 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7794 VideoFrameType::kVideoFrameDelta,
7795 VideoFrameType::kVideoFrameDelta}));
7796
7797 // Re-enable top layer.
7798 // Encoder should be re-initialized. Next frame should be key frame.
7799 config.simulcast_layers[number_layers - 1].active = true;
7800 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7801 sink_.SetNumExpectedLayers(number_layers);
7802 timestamp_ms += kFrameIntervalMs;
7803 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7804 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007805 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007806 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7807 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7808 VideoFrameType::kVideoFrameKey,
7809 VideoFrameType::kVideoFrameKey}));
7810
7811 // Top layer max rate change.
7812 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7813 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7814 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7815 sink_.SetNumExpectedLayers(number_layers);
7816 timestamp_ms += kFrameIntervalMs;
7817 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7818 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007819 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007820 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7821 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7822 VideoFrameType::kVideoFrameDelta,
7823 VideoFrameType::kVideoFrameDelta}));
7824
7825 // Top layer resolution change.
7826 // Encoder should be re-initialized. Next frame should be key frame.
7827 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7828 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7829 sink_.SetNumExpectedLayers(number_layers);
7830 timestamp_ms += kFrameIntervalMs;
7831 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7832 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007833 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007834 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7835 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7836 VideoFrameType::kVideoFrameKey,
7837 VideoFrameType::kVideoFrameKey}));
7838 video_stream_encoder_->Stop();
7839}
7840
Henrik Boström1124ed12021-02-25 10:30:39 +01007841TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7842 const int kFrameWidth = 1280;
7843 const int kFrameHeight = 720;
7844
7845 SetUp();
7846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007848
7849 // Capturing a frame should reconfigure the encoder and expose the encoder
7850 // resolution, which is the same as the input frame.
7851 int64_t timestamp_ms = kFrameIntervalMs;
7852 video_source_.IncomingCapturedFrame(
7853 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7854 WaitForEncodedFrame(timestamp_ms);
7855 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7856 EXPECT_THAT(video_source_.sink_wants().resolutions,
7857 ::testing::ElementsAreArray(
7858 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7859
7860 video_stream_encoder_->Stop();
7861}
7862
7863TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7864 // Pick downscale factors such that we never encode at full resolution - this
7865 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007866 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007867 // encoder should not ask for the frame resolution. This allows video frames
7868 // to have the appearence of one resolution but optimize its internal buffers
7869 // for what is actually encoded.
7870 const size_t kNumSimulcastLayers = 3u;
7871 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7872 const int kFrameWidth = 1280;
7873 const int kFrameHeight = 720;
7874 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7875 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7876 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7877 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7878 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7879 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7880
7881 VideoEncoderConfig config;
7882 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7883 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7884 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7885 config.simulcast_layers[i].active = true;
7886 }
7887 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007888 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007889 "VP8", /*max qp*/ 56, /*screencast*/ false,
7890 /*screenshare enabled*/ false);
7891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007892 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7893 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007894
7895 // Capture a frame with all layers active.
7896 int64_t timestamp_ms = kFrameIntervalMs;
7897 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7898 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7899 video_source_.IncomingCapturedFrame(
7900 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7901 WaitForEncodedFrame(timestamp_ms);
7902 // Expect encoded resolutions to match the expected simulcast layers.
7903 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7904 EXPECT_THAT(
7905 video_source_.sink_wants().resolutions,
7906 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7907
7908 // Capture a frame with one of the layers inactive.
7909 timestamp_ms += kFrameIntervalMs;
7910 config.simulcast_layers[2].active = false;
7911 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7912 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7913 video_source_.IncomingCapturedFrame(
7914 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7915 WaitForEncodedFrame(timestamp_ms);
7916
7917 // Expect encoded resolutions to match the expected simulcast layers.
7918 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7919 EXPECT_THAT(video_source_.sink_wants().resolutions,
7920 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7921
7922 // Capture a frame with all but one layer turned off.
7923 timestamp_ms += kFrameIntervalMs;
7924 config.simulcast_layers[1].active = false;
7925 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7926 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7927 video_source_.IncomingCapturedFrame(
7928 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7929 WaitForEncodedFrame(timestamp_ms);
7930
7931 // Expect encoded resolutions to match the expected simulcast layers.
7932 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7933 EXPECT_THAT(video_source_.sink_wants().resolutions,
7934 ::testing::ElementsAreArray({kLayer0Size}));
7935
7936 video_stream_encoder_->Stop();
7937}
7938
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007939TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007940 ResetEncoder("VP8", 1, 1, 1, false);
7941
Niels Möller8b692902021-06-14 12:04:57 +02007942 // Force encoder reconfig.
7943 video_source_.IncomingCapturedFrame(
7944 CreateFrame(1, codec_width_, codec_height_));
7945 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7946
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007947 // Set QP on encoded frame and pass the frame to encode complete callback.
7948 // Since QP is present QP parsing won't be triggered and the original value
7949 // should be kept.
7950 EncodedImage encoded_image;
7951 encoded_image.qp_ = 123;
7952 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7953 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7954 CodecSpecificInfo codec_info;
7955 codec_info.codecType = kVideoCodecVP8;
7956 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7957 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7958 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7959 video_stream_encoder_->Stop();
7960}
7961
7962TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007963 ResetEncoder("VP8", 1, 1, 1, false);
7964
Niels Möller8b692902021-06-14 12:04:57 +02007965 // Force encoder reconfig.
7966 video_source_.IncomingCapturedFrame(
7967 CreateFrame(1, codec_width_, codec_height_));
7968 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7969
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007970 // Pass an encoded frame without QP to encode complete callback. QP should be
7971 // parsed and set.
7972 EncodedImage encoded_image;
7973 encoded_image.qp_ = -1;
7974 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7975 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7976 CodecSpecificInfo codec_info;
7977 codec_info.codecType = kVideoCodecVP8;
7978 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7979 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7980 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7981 video_stream_encoder_->Stop();
7982}
7983
7984TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7985 webrtc::test::ScopedFieldTrials field_trials(
7986 "WebRTC-QpParsingKillSwitch/Enabled/");
7987
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007988 ResetEncoder("VP8", 1, 1, 1, false);
7989
Niels Möller8b692902021-06-14 12:04:57 +02007990 // Force encoder reconfig.
7991 video_source_.IncomingCapturedFrame(
7992 CreateFrame(1, codec_width_, codec_height_));
7993 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7994
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007995 EncodedImage encoded_image;
7996 encoded_image.qp_ = -1;
7997 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7998 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7999 CodecSpecificInfo codec_info;
8000 codec_info.codecType = kVideoCodecVP8;
8001 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8002 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8003 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8004 video_stream_encoder_->Stop();
8005}
8006
Sergey Silkind19e3b92021-03-16 10:05:30 +00008007TEST_F(VideoStreamEncoderTest,
8008 QualityScalingNotAllowed_QualityScalingDisabled) {
8009 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8010
8011 // Disable scaling settings in encoder info.
8012 fake_encoder_.SetQualityScaling(false);
8013 // Disable quality scaling in encoder config.
8014 video_encoder_config.is_quality_scaling_allowed = false;
8015 ConfigureEncoder(std::move(video_encoder_config));
8016
8017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008018 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008019
8020 test::FrameForwarder source;
8021 video_stream_encoder_->SetSource(
8022 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8023 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8025
8026 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8027 WaitForEncodedFrame(1);
8028 video_stream_encoder_->TriggerQualityLow();
8029 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8030
8031 video_stream_encoder_->Stop();
8032}
8033
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008034TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8035 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8036
8037 // Disable scaling settings in encoder info.
8038 fake_encoder_.SetQualityScaling(false);
8039 // Set QP trusted in encoder info.
8040 fake_encoder_.SetIsQpTrusted(true);
8041 // Enable quality scaling in encoder config.
8042 video_encoder_config.is_quality_scaling_allowed = false;
8043 ConfigureEncoder(std::move(video_encoder_config));
8044
8045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008046 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008047
8048 test::FrameForwarder source;
8049 video_stream_encoder_->SetSource(
8050 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8051 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8052 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8053
8054 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8055 WaitForEncodedFrame(1);
8056 video_stream_encoder_->TriggerQualityLow();
8057 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8058
8059 video_stream_encoder_->Stop();
8060}
8061
Shuhai Pengf2707702021-09-29 17:19:44 +08008062TEST_F(VideoStreamEncoderTest,
8063 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8064 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8065
8066 // Disable scaling settings in encoder info.
8067 fake_encoder_.SetQualityScaling(false);
8068 // Set QP trusted in encoder info.
8069 fake_encoder_.SetIsQpTrusted(true);
8070 // Enable quality scaling in encoder config.
8071 video_encoder_config.is_quality_scaling_allowed = false;
8072 ConfigureEncoder(std::move(video_encoder_config));
8073
8074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008075 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008076
8077 test::FrameForwarder source;
8078 video_stream_encoder_->SetSource(
8079 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8080 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8081 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8082
8083 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8084 WaitForEncodedFrame(1);
8085 video_stream_encoder_->TriggerQualityLow();
8086 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8087
8088 video_stream_encoder_->Stop();
8089}
8090
8091TEST_F(VideoStreamEncoderTest,
8092 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8093 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8094
8095 // Disable scaling settings in encoder info.
8096 fake_encoder_.SetQualityScaling(false);
8097 // Set QP trusted in encoder info.
8098 fake_encoder_.SetIsQpTrusted(false);
8099 // Enable quality scaling in encoder config.
8100 video_encoder_config.is_quality_scaling_allowed = false;
8101 ConfigureEncoder(std::move(video_encoder_config));
8102
8103 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008104 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008105
8106 test::FrameForwarder source;
8107 video_stream_encoder_->SetSource(
8108 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8109 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8110 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8111
8112 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8113 WaitForEncodedFrame(1);
8114 video_stream_encoder_->TriggerQualityLow();
8115 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8116
8117 video_stream_encoder_->Stop();
8118}
8119
8120TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8121 // Set QP trusted in encoder info.
8122 fake_encoder_.SetIsQpTrusted(false);
8123
8124 const int MinEncBitrateKbps = 30;
8125 const int MaxEncBitrateKbps = 100;
8126 const int MinStartBitrateKbp = 50;
8127 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8128 /*frame_size_pixels=*/codec_width_ * codec_height_,
8129 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8130 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8131 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8132
8133 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008134 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008135
8136 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8137
8138 VideoEncoderConfig video_encoder_config;
8139 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8140 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8141 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8142 MinEncBitrateKbps * 1000;
8143 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8144 kMaxPayloadLength);
8145
8146 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8147 WaitForEncodedFrame(1);
8148 EXPECT_EQ(
8149 MaxEncBitrateKbps,
8150 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8151 EXPECT_EQ(
8152 MinEncBitrateKbps,
8153 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8154
8155 video_stream_encoder_->Stop();
8156}
8157
8158TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8159 // Set QP trusted in encoder info.
8160 fake_encoder_.SetIsQpTrusted(false);
8161
8162 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8163 EncoderInfoSettings::
8164 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8165 codec_width_ * codec_height_,
8166 EncoderInfoSettings::
8167 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8168 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8169
8170 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8171 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8172 const int TargetEncBitrate = MaxEncBitrate;
8173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8174 DataRate::BitsPerSec(TargetEncBitrate),
8175 DataRate::BitsPerSec(TargetEncBitrate),
8176 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8177
8178 VideoEncoderConfig video_encoder_config;
8179 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8180 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8181 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8182 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8183 kMaxPayloadLength);
8184
8185 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8186 WaitForEncodedFrame(1);
8187 EXPECT_EQ(
8188 MaxEncBitrate / 1000,
8189 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8190 EXPECT_EQ(
8191 MinEncBitrate / 1000,
8192 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8193
8194 video_stream_encoder_->Stop();
8195}
8196
Sergey Silkind19e3b92021-03-16 10:05:30 +00008197#if !defined(WEBRTC_IOS)
8198// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8199// disabled by default on iOS.
8200TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8201 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8202
8203 // Disable scaling settings in encoder info.
8204 fake_encoder_.SetQualityScaling(false);
8205 // Enable quality scaling in encoder config.
8206 video_encoder_config.is_quality_scaling_allowed = true;
8207 ConfigureEncoder(std::move(video_encoder_config));
8208
8209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008210 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008211
8212 test::FrameForwarder source;
8213 video_stream_encoder_->SetSource(
8214 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8215 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8216 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8217
8218 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8219 WaitForEncodedFrame(1);
8220 video_stream_encoder_->TriggerQualityLow();
8221 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8222
8223 video_stream_encoder_->Stop();
8224}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008225
8226TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8227 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8228
8229 // Disable scaling settings in encoder info.
8230 fake_encoder_.SetQualityScaling(false);
8231 // Set QP trusted in encoder info.
8232 fake_encoder_.SetIsQpTrusted(true);
8233 // Enable quality scaling in encoder config.
8234 video_encoder_config.is_quality_scaling_allowed = true;
8235 ConfigureEncoder(std::move(video_encoder_config));
8236
8237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008238 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008239
8240 test::FrameForwarder source;
8241 video_stream_encoder_->SetSource(
8242 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8243 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8245
8246 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8247 WaitForEncodedFrame(1);
8248 video_stream_encoder_->TriggerQualityLow();
8249 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8250
8251 video_stream_encoder_->Stop();
8252}
Shuhai Pengf2707702021-09-29 17:19:44 +08008253
8254TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8255 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8256
8257 // Disable scaling settings in encoder info.
8258 fake_encoder_.SetQualityScaling(false);
8259 // Set QP not trusted in encoder info.
8260 fake_encoder_.SetIsQpTrusted(false);
8261 // Enable quality scaling in encoder config.
8262 video_encoder_config.is_quality_scaling_allowed = true;
8263 ConfigureEncoder(std::move(video_encoder_config));
8264
8265 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008266 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008267
8268 test::FrameForwarder source;
8269 video_stream_encoder_->SetSource(
8270 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8271 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8272 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8273
8274 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8275 WaitForEncodedFrame(1);
8276 video_stream_encoder_->TriggerQualityLow();
8277 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8278 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8279 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8280
8281 video_stream_encoder_->Stop();
8282}
8283
8284TEST_F(VideoStreamEncoderTest,
8285 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8286 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8287
8288 // Disable scaling settings in encoder info.
8289 fake_encoder_.SetQualityScaling(false);
8290 // Set QP trusted in encoder info.
8291 fake_encoder_.SetIsQpTrusted(true);
8292 // Enable quality scaling in encoder config.
8293 video_encoder_config.is_quality_scaling_allowed = true;
8294 ConfigureEncoder(std::move(video_encoder_config));
8295
8296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008297 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008298
8299 test::FrameForwarder source;
8300 video_stream_encoder_->SetSource(
8301 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8302 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8303 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8304
8305 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8306 WaitForEncodedFrame(1);
8307 video_stream_encoder_->TriggerQualityLow();
8308 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8309 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8310
8311 video_stream_encoder_->Stop();
8312}
8313
8314TEST_F(VideoStreamEncoderTest,
8315 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8316 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8317
8318 // Disable scaling settings in encoder info.
8319 fake_encoder_.SetQualityScaling(false);
8320 // Set QP trusted in encoder info.
8321 fake_encoder_.SetIsQpTrusted(false);
8322 // Enable quality scaling in encoder config.
8323 video_encoder_config.is_quality_scaling_allowed = true;
8324 ConfigureEncoder(std::move(video_encoder_config));
8325
8326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008327 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008328
8329 test::FrameForwarder source;
8330 video_stream_encoder_->SetSource(
8331 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8332 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8333 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8334
8335 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8336 WaitForEncodedFrame(1);
8337 video_stream_encoder_->TriggerQualityLow();
8338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8339
8340 video_stream_encoder_->Stop();
8341}
8342
Sergey Silkind19e3b92021-03-16 10:05:30 +00008343#endif
8344
Henrik Boström56db9ff2021-03-24 09:06:45 +01008345// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8346class VideoStreamEncoderWithRealEncoderTest
8347 : public VideoStreamEncoderTest,
8348 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8349 public:
8350 VideoStreamEncoderWithRealEncoderTest()
8351 : VideoStreamEncoderTest(),
8352 codec_type_(std::get<0>(GetParam())),
8353 allow_i420_conversion_(std::get<1>(GetParam())) {}
8354
8355 void SetUp() override {
8356 VideoStreamEncoderTest::SetUp();
8357 std::unique_ptr<VideoEncoder> encoder;
8358 switch (codec_type_) {
8359 case kVideoCodecVP8:
8360 encoder = VP8Encoder::Create();
8361 break;
8362 case kVideoCodecVP9:
8363 encoder = VP9Encoder::Create();
8364 break;
8365 case kVideoCodecAV1:
8366 encoder = CreateLibaomAv1Encoder();
8367 break;
8368 case kVideoCodecH264:
8369 encoder =
8370 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8371 break;
8372 case kVideoCodecMultiplex:
8373 mock_encoder_factory_for_multiplex_ =
8374 std::make_unique<MockVideoEncoderFactory>();
8375 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8376 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8377 .WillRepeatedly([] { return VP8Encoder::Create(); });
8378 encoder = std::make_unique<MultiplexEncoderAdapter>(
8379 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8380 false);
8381 break;
8382 default:
8383 RTC_NOTREACHED();
8384 }
8385 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8386 }
8387
8388 void TearDown() override {
8389 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008390 // Ensure `video_stream_encoder_` is destroyed before
8391 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008392 video_stream_encoder_.reset();
8393 VideoStreamEncoderTest::TearDown();
8394 }
8395
8396 protected:
8397 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8398 std::unique_ptr<VideoEncoder> encoder) {
8399 // Configure VSE to use the encoder.
8400 encoder_ = std::move(encoder);
8401 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8402 encoder_.get(), &encoder_selector_);
8403 video_send_config_.encoder_settings.encoder_factory =
8404 encoder_proxy_factory_.get();
8405 VideoEncoderConfig video_encoder_config;
8406 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8407 video_encoder_config_ = video_encoder_config.Copy();
8408 ConfigureEncoder(video_encoder_config_.Copy());
8409
8410 // Set bitrate to ensure frame is not dropped.
8411 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008412 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008413 }
8414
8415 const VideoCodecType codec_type_;
8416 const bool allow_i420_conversion_;
8417 NiceMock<MockEncoderSelector> encoder_selector_;
8418 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8419 std::unique_ptr<VideoEncoder> encoder_;
8420 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8421};
8422
8423TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8424 auto native_i420_frame = test::CreateMappableNativeFrame(
8425 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8426 video_source_.IncomingCapturedFrame(native_i420_frame);
8427 WaitForEncodedFrame(codec_width_, codec_height_);
8428
8429 auto mappable_native_buffer =
8430 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8431 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8432 mappable_native_buffer->GetMappedFramedBuffers();
8433 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8434 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8435 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8436 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8437}
8438
8439TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8440 auto native_nv12_frame = test::CreateMappableNativeFrame(
8441 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8442 video_source_.IncomingCapturedFrame(native_nv12_frame);
8443 WaitForEncodedFrame(codec_width_, codec_height_);
8444
8445 auto mappable_native_buffer =
8446 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8447 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8448 mappable_native_buffer->GetMappedFramedBuffers();
8449 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8450 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8451 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8452 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8453
8454 if (!allow_i420_conversion_) {
8455 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8456 }
8457}
8458
Erik Språng7444b192021-06-02 14:02:13 +02008459TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8460 if (codec_type_ == kVideoCodecMultiplex) {
8461 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8462 return;
8463 }
8464
8465 const size_t kNumSpatialLayers = 3u;
8466 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8467 const int kFrameWidth = 1280;
8468 const int kFrameHeight = 720;
8469 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8470 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8471 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8472 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8473 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8474 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8475
8476 VideoEncoderConfig config;
8477 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8478 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008479 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008480 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8481 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8482 vp9_settings.numberOfTemporalLayers = 3;
8483 vp9_settings.automaticResizeOn = false;
8484 config.encoder_specific_settings =
8485 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8486 vp9_settings);
8487 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8488 /*fps=*/30.0,
8489 /*first_active_layer=*/0,
8490 /*num_spatial_layers=*/3,
8491 /*num_temporal_layers=*/3,
8492 /*is_screenshare=*/false);
8493 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8494 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008495 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008496 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8497 /*fps=*/30.0,
8498 /*first_active_layer=*/0,
8499 /*num_spatial_layers=*/3,
8500 /*num_temporal_layers=*/3,
8501 /*is_screenshare=*/false);
8502 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8503 } else {
8504 // Simulcast for VP8/H264.
8505 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8506 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8507 config.simulcast_layers[i].scale_resolution_down_by =
8508 kDownscaleFactors[i];
8509 config.simulcast_layers[i].active = true;
8510 }
8511 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8512 // Turn off frame dropping to prevent flakiness.
8513 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8514 h264_settings.frameDroppingOn = false;
8515 config.encoder_specific_settings = rtc::make_ref_counted<
8516 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8517 }
8518 }
8519
8520 auto set_layer_active = [&](int layer_idx, bool active) {
8521 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8522 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8523 config.spatial_layers[layer_idx].active = active;
8524 } else {
8525 config.simulcast_layers[layer_idx].active = active;
8526 }
8527 };
8528
8529 config.video_stream_factory =
8530 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8531 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8532 /*screencast*/ false,
8533 /*screenshare enabled*/ false);
8534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008535 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8536 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008537
8538 // Capture a frame with all layers active.
8539 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8540 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8541 int64_t timestamp_ms = kFrameIntervalMs;
8542 video_source_.IncomingCapturedFrame(
8543 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8544
8545 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8546 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8547
8548 // Capture a frame with one of the layers inactive.
8549 set_layer_active(2, false);
8550 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8551 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8552 timestamp_ms += kFrameIntervalMs;
8553 video_source_.IncomingCapturedFrame(
8554 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8555 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8556
8557 // New target bitrates signaled based on lower resolution.
8558 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8560 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8561 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8562
8563 // Re-enable the top layer.
8564 set_layer_active(2, true);
8565 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8566 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8567 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8568
8569 // Bitrate target adjusted back up to enable HD layer...
8570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8571 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8572 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8573 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8574
8575 // ...then add a new frame.
8576 timestamp_ms += kFrameIntervalMs;
8577 video_source_.IncomingCapturedFrame(
8578 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8579 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8580 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8581
8582 video_stream_encoder_->Stop();
8583}
8584
Henrik Boström56db9ff2021-03-24 09:06:45 +01008585std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8586 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8587 VideoCodecType codec_type = std::get<0>(info.param);
8588 bool allow_i420_conversion = std::get<1>(info.param);
8589 std::string str;
8590 switch (codec_type) {
8591 case kVideoCodecGeneric:
8592 str = "Generic";
8593 break;
8594 case kVideoCodecVP8:
8595 str = "VP8";
8596 break;
8597 case kVideoCodecVP9:
8598 str = "VP9";
8599 break;
8600 case kVideoCodecAV1:
8601 str = "AV1";
8602 break;
8603 case kVideoCodecH264:
8604 str = "H264";
8605 break;
8606 case kVideoCodecMultiplex:
8607 str = "Multiplex";
8608 break;
8609 default:
8610 RTC_NOTREACHED();
8611 }
8612 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8613 return str;
8614}
8615
8616constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8617 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8618constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8619 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8620constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8621 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8622constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8623 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8624#if defined(WEBRTC_USE_H264)
8625constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8626 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8627
8628// The windows compiler does not tolerate #if statements inside the
8629// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8630// and without H264).
8631INSTANTIATE_TEST_SUITE_P(
8632 All,
8633 VideoStreamEncoderWithRealEncoderTest,
8634 ::testing::Values(kVP8DisallowConversion,
8635 kVP9DisallowConversion,
8636 kAV1AllowConversion,
8637 kMultiplexDisallowConversion,
8638 kH264AllowConversion),
8639 TestParametersVideoCodecAndAllowI420ConversionToString);
8640#else
8641INSTANTIATE_TEST_SUITE_P(
8642 All,
8643 VideoStreamEncoderWithRealEncoderTest,
8644 ::testing::Values(kVP8DisallowConversion,
8645 kVP9DisallowConversion,
8646 kAV1AllowConversion,
8647 kMultiplexDisallowConversion),
8648 TestParametersVideoCodecAndAllowI420ConversionToString);
8649#endif
8650
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008651class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8652 protected:
8653 void RunTest(const std::vector<VideoStream>& configs,
8654 const int expected_num_init_encode) {
8655 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008656 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008657 InsertFrameAndWaitForEncoded();
8658 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8659 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008660 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8661 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008662
8663 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8664 ConfigureEncoder(configs[1]);
8665 InsertFrameAndWaitForEncoded();
8666 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8667 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008668 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008669 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008670 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008671
8672 video_stream_encoder_->Stop();
8673 }
8674
8675 void ConfigureEncoder(const VideoStream& stream) {
8676 VideoEncoderConfig config;
8677 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8678 config.max_bitrate_bps = stream.max_bitrate_bps;
8679 config.simulcast_layers[0] = stream;
8680 config.video_stream_factory =
8681 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8682 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8683 /*conference_mode=*/false);
8684 video_stream_encoder_->ConfigureEncoder(std::move(config),
8685 kMaxPayloadLength);
8686 }
8687
8688 void OnBitrateUpdated(DataRate bitrate) {
8689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8690 bitrate, bitrate, bitrate, 0, 0, 0);
8691 }
8692
8693 void InsertFrameAndWaitForEncoded() {
8694 timestamp_ms_ += kFrameIntervalMs;
8695 video_source_.IncomingCapturedFrame(
8696 CreateFrame(timestamp_ms_, kWidth, kHeight));
8697 sink_.WaitForEncodedFrame(timestamp_ms_);
8698 }
8699
8700 void ExpectEqual(const VideoCodec& actual,
8701 const VideoStream& expected) const {
8702 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8703 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8704 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8705 static_cast<unsigned int>(expected.min_bitrate_bps));
8706 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8707 static_cast<unsigned int>(expected.max_bitrate_bps));
8708 EXPECT_EQ(actual.simulcastStream[0].width,
8709 kWidth / expected.scale_resolution_down_by);
8710 EXPECT_EQ(actual.simulcastStream[0].height,
8711 kHeight / expected.scale_resolution_down_by);
8712 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8713 expected.num_temporal_layers);
8714 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8715 }
8716
8717 VideoStream DefaultConfig() const {
8718 VideoStream stream;
8719 stream.max_framerate = 25;
8720 stream.min_bitrate_bps = 35000;
8721 stream.max_bitrate_bps = 900000;
8722 stream.scale_resolution_down_by = 1.0;
8723 stream.num_temporal_layers = 1;
8724 stream.bitrate_priority = 1.0;
8725 stream.scalability_mode = "";
8726 return stream;
8727 }
8728
8729 const int kWidth = 640;
8730 const int kHeight = 360;
8731 int64_t timestamp_ms_ = 0;
8732};
8733
8734TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8735 VideoStream config1 = DefaultConfig();
8736 VideoStream config2 = config1;
8737 config2.max_framerate++;
8738
8739 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8740}
8741
8742TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8743 VideoStream config1 = DefaultConfig();
8744 VideoStream config2 = config1;
8745 config2.min_bitrate_bps += 10000;
8746
8747 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8748}
8749
8750TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8751 VideoStream config1 = DefaultConfig();
8752 VideoStream config2 = config1;
8753 config2.max_bitrate_bps += 100000;
8754
8755 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8756}
8757
8758TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8759 VideoStream config1 = DefaultConfig();
8760 VideoStream config2 = config1;
8761 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8762
8763 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8764}
8765
8766TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8767 VideoStream config1 = DefaultConfig();
8768 VideoStream config2 = config1;
8769 config2.scale_resolution_down_by *= 2;
8770
8771 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8772}
8773
8774TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8775 VideoStream config1 = DefaultConfig();
8776 VideoStream config2 = config1;
8777 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8778
8779 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8780}
8781
8782TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8783 VideoStream config1 = DefaultConfig();
8784 VideoStream config2 = config1;
8785 config2.scalability_mode = "L1T2";
8786
8787 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8788}
8789
Markus Handellb4e96d42021-11-05 12:00:55 +01008790TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8791 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8792 auto* adapter_ptr = adapter.get();
8793 SimpleVideoStreamEncoderFactory factory;
8794 auto video_stream_encoder = factory.Create(std::move(adapter));
8795
8796 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true));
8797 VideoEncoderConfig config;
8798 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8799 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
8800 Mock::VerifyAndClearExpectations(adapter_ptr);
8801
8802 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false));
8803 VideoEncoderConfig config2;
8804 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8805 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
8806}
8807
8808TEST(VideoStreamEncoderFrameCadenceTest,
8809 ForwardsFramesIntoFrameCadenceAdapter) {
8810 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8811 auto* adapter_ptr = adapter.get();
8812 test::FrameForwarder video_source;
8813 SimpleVideoStreamEncoderFactory factory;
8814 auto video_stream_encoder = factory.Create(std::move(adapter));
8815 video_stream_encoder->SetSource(
8816 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8817
8818 EXPECT_CALL(*adapter_ptr, OnFrame);
8819 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8820 video_source.IncomingCapturedFrame(
8821 VideoFrame::Builder()
8822 .set_video_frame_buffer(std::move(buffer))
8823 .set_ntp_time_ms(0)
8824 .set_timestamp_ms(0)
8825 .set_rotation(kVideoRotation_0)
8826 .build());
8827}
8828
perkj26091b12016-09-01 01:17:40 -07008829} // namespace webrtc