blob: f7a36216c2b084d3d70a0dda61b014f6b3613950 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020040#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020043#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010044#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020045#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020060using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020062using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020063using ::testing::Ge;
64using ::testing::Gt;
65using ::testing::Le;
66using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010067using ::testing::Matcher;
68using ::testing::NiceMock;
69using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010070using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020071using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080072
perkj803d97f2016-11-01 11:45:46 -070073namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010075const int kQpLow = 1;
76const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020077const int kMinFramerateFps = 2;
78const int kMinBalancedFramerateFps = 7;
79const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080080const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020082const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010083const uint32_t kSimulcastTargetBitrateBps = 3150000;
84const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080085const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070086const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020088const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020089const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
91const VideoEncoder::ResolutionBitrateLimits
92 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020094uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
95 0x00, 0x00, 0x03, 0x03, 0xF4,
96 0x05, 0x03, 0xC7, 0xE0, 0x1B,
97 0x41, 0x10, 0x8D, 0x00};
98
perkj803d97f2016-11-01 11:45:46 -070099class TestBuffer : public webrtc::I420Buffer {
100 public:
101 TestBuffer(rtc::Event* event, int width, int height)
102 : I420Buffer(width, height), event_(event) {}
103
104 private:
105 friend class rtc::RefCountedObject<TestBuffer>;
106 ~TestBuffer() override {
107 if (event_)
108 event_->Set();
109 }
110 rtc::Event* const event_;
111};
112
Noah Richards51db4212019-06-12 06:59:12 -0700113// A fake native buffer that can't be converted to I420.
114class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
115 public:
116 FakeNativeBuffer(rtc::Event* event, int width, int height)
117 : event_(event), width_(width), height_(height) {}
118 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
119 int width() const override { return width_; }
120 int height() const override { return height_; }
121 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
122 return nullptr;
123 }
124
125 private:
126 friend class rtc::RefCountedObject<FakeNativeBuffer>;
127 ~FakeNativeBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132 const int width_;
133 const int height_;
134};
135
Evan Shrubsole895556e2020-10-05 09:15:13 +0200136// A fake native buffer that is backed by an NV12 buffer.
137class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
140 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
141
142 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
143 int width() const override { return nv12_buffer_->width(); }
144 int height() const override { return nv12_buffer_->height(); }
145 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
146 return nv12_buffer_->ToI420();
147 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200148 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
149 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
150 if (absl::c_find(types, Type::kNV12) != types.end()) {
151 return nv12_buffer_;
152 }
153 return nullptr;
154 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200155 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
156
157 private:
158 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
159 ~FakeNV12NativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
164 rtc::Event* const event_;
165};
166
Niels Möller7dc26b72017-12-06 10:27:48 +0100167class CpuOveruseDetectorProxy : public OveruseFrameDetector {
168 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200169 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
170 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200171 last_target_framerate_fps_(-1),
172 framerate_updated_event_(true /* manual_reset */,
173 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100174 virtual ~CpuOveruseDetectorProxy() {}
175
176 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200177 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100178 last_target_framerate_fps_ = framerate_fps;
179 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200180 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100181 }
182
183 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200184 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100185 return last_target_framerate_fps_;
186 }
187
Niels Möller4db138e2018-04-19 09:04:13 +0200188 CpuOveruseOptions GetOptions() { return options_; }
189
Henrik Boström381d1092020-05-12 18:49:07 +0200190 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
191
Niels Möller7dc26b72017-12-06 10:27:48 +0100192 private:
Markus Handella3765182020-07-08 13:13:32 +0200193 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100194 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200195 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100196};
197
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200198class FakeVideoSourceRestrictionsListener
199 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200200 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200201 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200202 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200203 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200204 RTC_DCHECK(was_restrictions_updated_);
205 }
206
207 rtc::Event* restrictions_updated_event() {
208 return &restrictions_updated_event_;
209 }
210
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200211 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200212 void OnVideoSourceRestrictionsUpdated(
213 VideoSourceRestrictions restrictions,
214 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200215 rtc::scoped_refptr<Resource> reason,
216 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200217 was_restrictions_updated_ = true;
218 restrictions_updated_event_.Set();
219 }
220
221 private:
222 bool was_restrictions_updated_;
223 rtc::Event restrictions_updated_event_;
224};
225
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200226auto WantsFps(Matcher<int> fps_matcher) {
227 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
228 fps_matcher);
229}
230
231auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
232 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
233 AllOf(max_pixel_matcher, Gt(0)));
234}
235
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200236auto ResolutionMax() {
237 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
240 Eq(absl::nullopt)));
241}
242
243auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245}
246
247auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200249}
250
251auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200253}
254
255auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200256 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200257}
258
259auto FpsMaxResolutionMax() {
260 return AllOf(FpsMax(), ResolutionMax());
261}
262
263auto UnlimitedSinkWants() {
264 return AllOf(FpsUnlimited(), ResolutionMax());
265}
266
267auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
268 Matcher<int> fps_range_matcher;
269
270 if (last_frame_pixels <= 320 * 240) {
271 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200272 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200273 fps_range_matcher = AllOf(Ge(10), Le(15));
274 } else if (last_frame_pixels <= 640 * 480) {
275 fps_range_matcher = Ge(15);
276 } else {
277 fps_range_matcher = Eq(kDefaultFramerate);
278 }
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_range_matcher);
281}
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
284 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
285 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
286}
287
288auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
289 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
290}
291
292auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
293 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
294}
295
296auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
299}
300
301auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
304}
305
306auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
307 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
308 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
309}
310
311auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
312 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
313 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
314}
315
mflodmancc3d4422017-08-03 08:27:51 -0700316class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700317 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200318 VideoStreamEncoderUnderTest(TimeController* time_controller,
319 TaskQueueFactory* task_queue_factory,
320 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100321 const VideoStreamEncoderSettings& settings,
322 VideoStreamEncoder::BitrateAllocationCallbackType
323 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200324 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100325 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200326 stats_proxy,
327 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200328 std::unique_ptr<OveruseFrameDetector>(
329 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100330 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100331 task_queue_factory,
332 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200333 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200334 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200335 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200336 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200337 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200338 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200339 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200340 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100341 }
perkj803d97f2016-11-01 11:45:46 -0700342
Henrik Boström381d1092020-05-12 18:49:07 +0200343 void SetSourceAndWaitForRestrictionsUpdated(
344 rtc::VideoSourceInterface<VideoFrame>* source,
345 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200346 FakeVideoSourceRestrictionsListener listener;
347 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200348 SetSource(source, degradation_preference);
349 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200350 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200351 }
352
353 void SetSourceAndWaitForFramerateUpdated(
354 rtc::VideoSourceInterface<VideoFrame>* source,
355 const DegradationPreference& degradation_preference) {
356 overuse_detector_proxy_->framerate_updated_event()->Reset();
357 SetSource(source, degradation_preference);
358 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
359 }
360
361 void OnBitrateUpdatedAndWaitForManagedResources(
362 DataRate target_bitrate,
363 DataRate stable_target_bitrate,
364 DataRate link_allocation,
365 uint8_t fraction_lost,
366 int64_t round_trip_time_ms,
367 double cwnd_reduce_ratio) {
368 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
369 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
370 // Bitrate is updated on the encoder queue.
371 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200372 }
373
kthelgason2fc52542017-03-03 00:24:41 -0800374 // This is used as a synchronisation mechanism, to make sure that the
375 // encoder queue is not blocked before we start sending it frames.
376 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100377 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200378 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800379 ASSERT_TRUE(event.Wait(5000));
380 }
381
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200383 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200384 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200385 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200386 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200387 event.Set();
388 });
389 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200390 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200391 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200392
Henrik Boström91aa7322020-04-28 12:24:33 +0200393 void TriggerCpuUnderuse() {
394 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200395 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200396 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200397 event.Set();
398 });
399 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200400 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200401 }
kthelgason876222f2016-11-29 01:44:11 -0800402
Henrik Boström91aa7322020-04-28 12:24:33 +0200403 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200404 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200405 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200406 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200407 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 event.Set();
409 });
410 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200411 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200412 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200413 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200415 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200416 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 event.Set();
418 });
419 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200421 }
422
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200423 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100424 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200425 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
426 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200427 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700428};
429
Noah Richards51db4212019-06-12 06:59:12 -0700430// Simulates simulcast behavior and makes highest stream resolutions divisible
431// by 4.
432class CroppingVideoStreamFactory
433 : public VideoEncoderConfig::VideoStreamFactoryInterface {
434 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200435 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700436
437 private:
438 std::vector<VideoStream> CreateEncoderStreams(
439 int width,
440 int height,
441 const VideoEncoderConfig& encoder_config) override {
442 std::vector<VideoStream> streams = test::CreateVideoStreams(
443 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700444 return streams;
445 }
Noah Richards51db4212019-06-12 06:59:12 -0700446};
447
sprangb1ca0732017-02-01 08:38:12 -0800448class AdaptingFrameForwarder : public test::FrameForwarder {
449 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 explicit AdaptingFrameForwarder(TimeController* time_controller)
451 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700452 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800453
454 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200455 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800456 adaptation_enabled_ = enabled;
457 }
458
asaperssonfab67072017-04-04 05:51:49 -0700459 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200460 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800461 return adaptation_enabled_;
462 }
463
asapersson09f05612017-05-15 23:40:18 -0700464 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200465 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700466 return last_wants_;
467 }
468
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200469 absl::optional<int> last_sent_width() const { return last_width_; }
470 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800471
sprangb1ca0732017-02-01 08:38:12 -0800472 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200473 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
474 time_controller_->AdvanceTime(TimeDelta::Millis(0));
475
sprangb1ca0732017-02-01 08:38:12 -0800476 int cropped_width = 0;
477 int cropped_height = 0;
478 int out_width = 0;
479 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700480 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200481 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
482 << "w=" << video_frame.width()
483 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700484 if (adapter_.AdaptFrameResolution(
485 video_frame.width(), video_frame.height(),
486 video_frame.timestamp_us() * 1000, &cropped_width,
487 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100488 VideoFrame adapted_frame =
489 VideoFrame::Builder()
490 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
491 nullptr, out_width, out_height))
492 .set_timestamp_rtp(99)
493 .set_timestamp_ms(99)
494 .set_rotation(kVideoRotation_0)
495 .build();
sprangc5d62e22017-04-02 23:53:04 -0700496 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100497 if (video_frame.has_update_rect()) {
498 adapted_frame.set_update_rect(
499 video_frame.update_rect().ScaleWithFrame(
500 video_frame.width(), video_frame.height(), 0, 0,
501 video_frame.width(), video_frame.height(), out_width,
502 out_height));
503 }
sprangc5d62e22017-04-02 23:53:04 -0700504 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800505 last_width_.emplace(adapted_frame.width());
506 last_height_.emplace(adapted_frame.height());
507 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200508 last_width_ = absl::nullopt;
509 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700510 }
sprangb1ca0732017-02-01 08:38:12 -0800511 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800513 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800514 last_width_.emplace(video_frame.width());
515 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800516 }
517 }
518
519 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
520 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200522 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100523 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200524 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800525 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200526
527 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800528 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200529 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
530 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200531 absl::optional<int> last_width_;
532 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800533};
sprangc5d62e22017-04-02 23:53:04 -0700534
Niels Möller213618e2018-07-24 09:29:58 +0200535// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700536class MockableSendStatisticsProxy : public SendStatisticsProxy {
537 public:
538 MockableSendStatisticsProxy(Clock* clock,
539 const VideoSendStream::Config& config,
540 VideoEncoderConfig::ContentType content_type)
541 : SendStatisticsProxy(clock, config, content_type) {}
542
543 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200544 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700545 if (mock_stats_)
546 return *mock_stats_;
547 return SendStatisticsProxy::GetStats();
548 }
549
Niels Möller213618e2018-07-24 09:29:58 +0200550 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200551 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200552 if (mock_stats_)
553 return mock_stats_->input_frame_rate;
554 return SendStatisticsProxy::GetInputFrameRate();
555 }
sprangc5d62e22017-04-02 23:53:04 -0700556 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200557 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700558 mock_stats_.emplace(stats);
559 }
560
561 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200562 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700563 mock_stats_.reset();
564 }
565
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200566 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
567 on_frame_dropped_ = std::move(callback);
568 }
569
sprangc5d62e22017-04-02 23:53:04 -0700570 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200571 void OnFrameDropped(DropReason reason) override {
572 SendStatisticsProxy::OnFrameDropped(reason);
573 if (on_frame_dropped_)
574 on_frame_dropped_(reason);
575 }
576
Markus Handella3765182020-07-08 13:13:32 +0200577 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200578 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200579 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700580};
581
philipel9b058032020-02-10 11:30:00 +0100582class MockEncoderSelector
583 : public VideoEncoderFactory::EncoderSelectorInterface {
584 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200585 MOCK_METHOD(void,
586 OnCurrentEncoder,
587 (const SdpVideoFormat& format),
588 (override));
589 MOCK_METHOD(absl::optional<SdpVideoFormat>,
590 OnAvailableBitrate,
591 (const DataRate& rate),
592 (override));
593 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100594};
595
perkj803d97f2016-11-01 11:45:46 -0700596} // namespace
597
mflodmancc3d4422017-08-03 08:27:51 -0700598class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700599 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200600 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700601
mflodmancc3d4422017-08-03 08:27:51 -0700602 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700603 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700604 codec_width_(320),
605 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200606 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200608 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700609 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700611 video_send_config_,
612 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200613 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700614
615 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700616 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700617 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200618 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800619 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200620 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200621 video_send_config_.rtp.payload_name = "FAKE";
622 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700623
Per512ecb32016-09-23 15:52:06 +0200624 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200625 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200626 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
627 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
628 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100629 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700630
Niels Möllerf1338562018-04-26 09:51:47 +0200631 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800632 }
633
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100634 void ConfigureEncoder(
635 VideoEncoderConfig video_encoder_config,
636 VideoStreamEncoder::BitrateAllocationCallbackType
637 allocation_callback_type =
638 VideoStreamEncoder::BitrateAllocationCallbackType::
639 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700640 if (video_stream_encoder_)
641 video_stream_encoder_->Stop();
642 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200643 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100644 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700645 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700647 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700648 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
649 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200650 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800652 }
653
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100654 void ResetEncoder(const std::string& payload_name,
655 size_t num_streams,
656 size_t num_temporal_layers,
657 unsigned char num_spatial_layers,
658 bool screenshare,
659 VideoStreamEncoder::BitrateAllocationCallbackType
660 allocation_callback_type =
661 VideoStreamEncoder::BitrateAllocationCallbackType::
662 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200663 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800664
665 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200666 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
667 num_streams, &video_encoder_config);
668 for (auto& layer : video_encoder_config.simulcast_layers) {
669 layer.num_temporal_layers = num_temporal_layers;
670 layer.max_framerate = kDefaultFramerate;
671 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100672 video_encoder_config.max_bitrate_bps =
673 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700674 video_encoder_config.content_type =
675 screenshare ? VideoEncoderConfig::ContentType::kScreen
676 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700677 if (payload_name == "VP9") {
678 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
679 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200680 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700681 video_encoder_config.encoder_specific_settings =
682 new rtc::RefCountedObject<
683 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
684 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100685 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700686 }
687
sprang57c2fff2017-01-16 06:24:02 -0800688 VideoFrame CreateFrame(int64_t ntp_time_ms,
689 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100690 VideoFrame frame =
691 VideoFrame::Builder()
692 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
693 destruction_event, codec_width_, codec_height_))
694 .set_timestamp_rtp(99)
695 .set_timestamp_ms(99)
696 .set_rotation(kVideoRotation_0)
697 .build();
sprang57c2fff2017-01-16 06:24:02 -0800698 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700699 return frame;
700 }
701
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100702 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
703 rtc::Event* destruction_event,
704 int offset_x) const {
705 VideoFrame frame =
706 VideoFrame::Builder()
707 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
708 destruction_event, codec_width_, codec_height_))
709 .set_timestamp_rtp(99)
710 .set_timestamp_ms(99)
711 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100712 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100713 .build();
714 frame.set_ntp_time_ms(ntp_time_ms);
715 return frame;
716 }
717
sprang57c2fff2017-01-16 06:24:02 -0800718 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100719 VideoFrame frame =
720 VideoFrame::Builder()
721 .set_video_frame_buffer(
722 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
723 .set_timestamp_rtp(99)
724 .set_timestamp_ms(99)
725 .set_rotation(kVideoRotation_0)
726 .build();
sprang57c2fff2017-01-16 06:24:02 -0800727 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700728 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700729 return frame;
730 }
731
Evan Shrubsole895556e2020-10-05 09:15:13 +0200732 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
733 VideoFrame frame =
734 VideoFrame::Builder()
735 .set_video_frame_buffer(NV12Buffer::Create(width, height))
736 .set_timestamp_rtp(99)
737 .set_timestamp_ms(99)
738 .set_rotation(kVideoRotation_0)
739 .build();
740 frame.set_ntp_time_ms(ntp_time_ms);
741 frame.set_timestamp_us(ntp_time_ms * 1000);
742 return frame;
743 }
744
Noah Richards51db4212019-06-12 06:59:12 -0700745 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int width,
748 int height) const {
749 VideoFrame frame =
750 VideoFrame::Builder()
751 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
752 destruction_event, width, height))
753 .set_timestamp_rtp(99)
754 .set_timestamp_ms(99)
755 .set_rotation(kVideoRotation_0)
756 .build();
757 frame.set_ntp_time_ms(ntp_time_ms);
758 return frame;
759 }
760
Evan Shrubsole895556e2020-10-05 09:15:13 +0200761 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
762 rtc::Event* destruction_event,
763 int width,
764 int height) const {
765 VideoFrame frame = VideoFrame::Builder()
766 .set_video_frame_buffer(
767 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
768 destruction_event, width, height))
769 .set_timestamp_rtp(99)
770 .set_timestamp_ms(99)
771 .set_rotation(kVideoRotation_0)
772 .build();
773 frame.set_ntp_time_ms(ntp_time_ms);
774 return frame;
775 }
776
Noah Richards51db4212019-06-12 06:59:12 -0700777 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
778 rtc::Event* destruction_event) const {
779 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
780 codec_height_);
781 }
782
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100783 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100785 DataRate::BitsPerSec(kTargetBitrateBps),
786 DataRate::BitsPerSec(kTargetBitrateBps),
787 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100788
789 video_source_.IncomingCapturedFrame(
790 CreateFrame(1, codec_width_, codec_height_));
791 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200792 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100793 }
794
sprang4847ae62017-06-27 07:06:52 -0700795 void WaitForEncodedFrame(int64_t expected_ntp_time) {
796 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200797 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700798 }
799
800 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
801 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200802 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700803 return ok;
804 }
805
806 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
807 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200808 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700809 }
810
811 void ExpectDroppedFrame() {
812 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200813 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 }
815
816 bool WaitForFrame(int64_t timeout_ms) {
817 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200818 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700819 return ok;
820 }
821
perkj26091b12016-09-01 01:17:40 -0700822 class TestEncoder : public test::FakeEncoder {
823 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 explicit TestEncoder(TimeController* time_controller)
825 : FakeEncoder(time_controller->GetClock()),
826 time_controller_(time_controller) {
827 RTC_DCHECK(time_controller_);
828 }
perkj26091b12016-09-01 01:17:40 -0700829
asaperssonfab67072017-04-04 05:51:49 -0700830 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200831 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700832 return config_;
833 }
834
835 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200836 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700837 block_next_encode_ = true;
838 }
839
Erik Språngaed30702018-11-05 12:57:17 +0100840 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200841 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200842 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100843 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100845 info.scaling_settings = VideoEncoder::ScalingSettings(
846 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100847 }
848 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 for (int i = 0; i < kMaxSpatialLayers; ++i) {
850 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100851 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100852 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100853 for (int tid = 0; tid < num_layers; ++tid)
854 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100855 }
856 }
Erik Språngaed30702018-11-05 12:57:17 +0100857 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200858
859 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100860 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200861 info.apply_alignment_to_all_simulcast_layers =
862 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200863 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100864 return info;
kthelgason876222f2016-11-29 01:44:11 -0800865 }
866
Erik Språngb7cb7b52019-02-26 15:52:33 +0100867 int32_t RegisterEncodeCompleteCallback(
868 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200869 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100870 encoded_image_callback_ = callback;
871 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
872 }
873
perkjfa10b552016-10-02 23:45:26 -0700874 void ContinueEncode() { continue_encode_event_.Set(); }
875
876 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
877 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200878 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700879 EXPECT_EQ(timestamp_, timestamp);
880 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
881 }
882
kthelgason2fc52542017-03-03 00:24:41 -0800883 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200884 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800885 quality_scaling_ = b;
886 }
kthelgasonad9010c2017-02-14 00:46:51 -0800887
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100888 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200889 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100890 requested_resolution_alignment_ = requested_resolution_alignment;
891 }
892
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200893 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
894 MutexLock lock(&local_mutex_);
895 apply_alignment_to_all_simulcast_layers_ = b;
896 }
897
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100898 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200899 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100900 is_hardware_accelerated_ = is_hardware_accelerated;
901 }
902
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100903 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
904 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100906 temporal_layers_supported_[spatial_idx] = supported;
907 }
908
Sergey Silkin6456e352019-07-08 17:56:40 +0200909 void SetResolutionBitrateLimits(
910 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200911 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200912 resolution_bitrate_limits_ = thresholds;
913 }
914
sprangfe627f32017-03-29 08:24:59 -0700915 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700917 force_init_encode_failed_ = force_failure;
918 }
919
Niels Möller6bb5ab92019-01-11 11:11:10 +0100920 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100922 rate_factor_ = rate_factor;
923 }
924
Erik Språngd7329ca2019-02-21 21:19:53 +0100925 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100927 return last_framerate_;
928 }
929
Erik Språngd7329ca2019-02-21 21:19:53 +0100930 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200931 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100932 return last_update_rect_;
933 }
934
Niels Möller87e2d782019-03-07 10:18:23 +0100935 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200936 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100937 return last_frame_types_;
938 }
939
940 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100941 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100942 keyframe ? VideoFrameType::kVideoFrameKey
943 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100944 {
Markus Handella3765182020-07-08 13:13:32 +0200945 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100946 last_frame_types_ = frame_type;
947 }
Niels Möllerb859b322019-03-07 12:40:01 +0100948 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100949 }
950
Erik Språngb7cb7b52019-02-26 15:52:33 +0100951 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200952 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200953 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100954 }
955
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200956 void SetEncodedImageData(
957 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200958 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200959 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200960 }
961
Erik Språngd7329ca2019-02-21 21:19:53 +0100962 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 expect_null_frame_ = true;
965 }
966
Erik Språng5056af02019-09-02 15:53:11 +0200967 absl::optional<VideoEncoder::RateControlParameters>
968 GetAndResetLastRateControlSettings() {
969 auto settings = last_rate_control_settings_;
970 last_rate_control_settings_.reset();
971 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100972 }
973
Evan Shrubsole895556e2020-10-05 09:15:13 +0200974 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
975 MutexLock lock(&local_mutex_);
976 return last_input_pixel_format_;
977 }
978
Sergey Silkin5ee69672019-07-02 14:18:34 +0200979 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200980 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200981 return num_encoder_initializations_;
982 }
983
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200984 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200985 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200986 return num_set_rates_;
987 }
988
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200989 VideoCodec video_codec() const {
990 MutexLock lock(&local_mutex_);
991 return video_codec_;
992 }
993
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200994 void SetPreferredPixelFormats(
995 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
996 pixel_formats) {
997 MutexLock lock(&local_mutex_);
998 preferred_pixel_formats_ = std::move(pixel_formats);
999 }
1000
perkjfa10b552016-10-02 23:45:26 -07001001 private:
perkj26091b12016-09-01 01:17:40 -07001002 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001003 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001004 bool block_encode;
1005 {
Markus Handella3765182020-07-08 13:13:32 +02001006 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001007 if (expect_null_frame_) {
1008 EXPECT_EQ(input_image.timestamp(), 0u);
1009 EXPECT_EQ(input_image.width(), 1);
1010 last_frame_types_ = *frame_types;
1011 expect_null_frame_ = false;
1012 } else {
1013 EXPECT_GT(input_image.timestamp(), timestamp_);
1014 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1015 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1016 }
perkj26091b12016-09-01 01:17:40 -07001017
1018 timestamp_ = input_image.timestamp();
1019 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001020 last_input_width_ = input_image.width();
1021 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001022 block_encode = block_next_encode_;
1023 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001024 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001025 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001026 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001027 }
Niels Möllerb859b322019-03-07 12:40:01 +01001028 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001029 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001030 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001031
perkj26091b12016-09-01 01:17:40 -07001032 return result;
1033 }
1034
Niels Möller08ae7ce2020-09-23 15:58:12 +02001035 CodecSpecificInfo EncodeHook(
1036 EncodedImage& encoded_image,
1037 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001038 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001039 {
1040 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001041 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001042 }
1043 MutexLock lock(&local_mutex_);
1044 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001045 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001046 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001047 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001048 }
1049
sprangfe627f32017-03-29 08:24:59 -07001050 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001051 const Settings& settings) override {
1052 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001053
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001055 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001056
1057 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001058 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001059
Erik Språng82fad3d2018-03-21 09:57:23 +01001060 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001061 // Simulate setting up temporal layers, in order to validate the life
1062 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001063 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001064 frame_buffer_controller_ =
1065 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001066 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001067 if (force_init_encode_failed_) {
1068 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001069 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001070 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001071
Erik Språngb7cb7b52019-02-26 15:52:33 +01001072 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001073 return res;
1074 }
1075
Erik Språngb7cb7b52019-02-26 15:52:33 +01001076 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001077 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1079 initialized_ = EncoderState::kUninitialized;
1080 return FakeEncoder::Release();
1081 }
1082
Erik Språng16cb8f52019-04-12 13:59:09 +02001083 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001085 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001086 VideoBitrateAllocation adjusted_rate_allocation;
1087 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1088 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001089 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001090 adjusted_rate_allocation.SetBitrate(
1091 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001092 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001093 rate_factor_));
1094 }
1095 }
1096 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001097 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001098 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001099 RateControlParameters adjusted_paramters = parameters;
1100 adjusted_paramters.bitrate = adjusted_rate_allocation;
1101 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001102 }
1103
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001104 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001105 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001106 enum class EncoderState {
1107 kUninitialized,
1108 kInitializationFailed,
1109 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001110 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1111 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001112 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001113 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1114 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1115 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1116 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1117 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1118 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001119 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1120 false;
Markus Handella3765182020-07-08 13:13:32 +02001121 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001122 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1123 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001124 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001125 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001126 absl::optional<bool>
1127 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001128 local_mutex_);
1129 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1130 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1131 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001132 absl::optional<VideoEncoder::RateControlParameters>
1133 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001134 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1135 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001136 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001137 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001138 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1139 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001140 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001141 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001142 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001143 RTC_GUARDED_BY(local_mutex_);
1144 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001145 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001146 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1147 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001148 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1149 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001150 };
1151
mflodmancc3d4422017-08-03 08:27:51 -07001152 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001153 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001154 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1155 : time_controller_(time_controller), test_encoder_(test_encoder) {
1156 RTC_DCHECK(time_controller_);
1157 }
perkj26091b12016-09-01 01:17:40 -07001158
perkj26091b12016-09-01 01:17:40 -07001159 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001160 EXPECT_TRUE(
1161 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1162 }
1163
1164 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1165 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001166 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001167 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001168 return false;
perkj26091b12016-09-01 01:17:40 -07001169 {
Markus Handella3765182020-07-08 13:13:32 +02001170 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001171 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001172 }
1173 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001174 return true;
perkj26091b12016-09-01 01:17:40 -07001175 }
1176
sprangb1ca0732017-02-01 08:38:12 -08001177 void WaitForEncodedFrame(uint32_t expected_width,
1178 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001179 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001180 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001181 }
1182
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001183 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001184 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001185 uint32_t width = 0;
1186 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001187 {
Markus Handella3765182020-07-08 13:13:32 +02001188 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001189 width = last_width_;
1190 height = last_height_;
1191 }
1192 EXPECT_EQ(expected_height, height);
1193 EXPECT_EQ(expected_width, width);
1194 }
1195
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001196 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1197 VideoRotation rotation;
1198 {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001200 rotation = last_rotation_;
1201 }
1202 EXPECT_EQ(expected_rotation, rotation);
1203 }
1204
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001205 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001206
sprangc5d62e22017-04-02 23:53:04 -07001207 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001208 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1209 bool ret = encoded_frame_event_.Wait(timeout_ms);
1210 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1211 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001212 }
1213
perkj26091b12016-09-01 01:17:40 -07001214 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001215 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001216 expect_frames_ = false;
1217 }
1218
asaperssonfab67072017-04-04 05:51:49 -07001219 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001220 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001221 return number_of_reconfigurations_;
1222 }
1223
asaperssonfab67072017-04-04 05:51:49 -07001224 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001225 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001226 return min_transmit_bitrate_bps_;
1227 }
1228
Erik Språngd7329ca2019-02-21 21:19:53 +01001229 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001230 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001231 num_expected_layers_ = num_layers;
1232 }
1233
Erik Språngb7cb7b52019-02-26 15:52:33 +01001234 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001235 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001236 return last_capture_time_ms_;
1237 }
1238
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001239 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001240 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001241 return std::move(last_encoded_image_data_);
1242 }
1243
Per Kjellanderdcef6412020-10-07 15:09:05 +02001244 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1245 MutexLock lock(&mutex_);
1246 return last_bitrate_allocation_;
1247 }
1248
1249 int number_of_bitrate_allocations() const {
1250 MutexLock lock(&mutex_);
1251 return number_of_bitrate_allocations_;
1252 }
1253
Per Kjellandera9434842020-10-15 17:53:22 +02001254 VideoLayersAllocation GetLastVideoLayersAllocation() {
1255 MutexLock lock(&mutex_);
1256 return last_layers_allocation_;
1257 }
1258
1259 int number_of_layers_allocations() const {
1260 MutexLock lock(&mutex_);
1261 return number_of_layers_allocations_;
1262 }
1263
perkj26091b12016-09-01 01:17:40 -07001264 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001265 Result OnEncodedImage(
1266 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001267 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001269 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001270 last_encoded_image_data_ = std::vector<uint8_t>(
1271 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001272 uint32_t timestamp = encoded_image.Timestamp();
1273 if (last_timestamp_ != timestamp) {
1274 num_received_layers_ = 1;
1275 } else {
1276 ++num_received_layers_;
1277 }
1278 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001279 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001280 last_width_ = encoded_image._encodedWidth;
1281 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001282 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001283 if (num_received_layers_ == num_expected_layers_) {
1284 encoded_frame_event_.Set();
1285 }
sprangb1ca0732017-02-01 08:38:12 -08001286 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001287 }
1288
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001289 void OnEncoderConfigurationChanged(
1290 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001291 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001292 VideoEncoderConfig::ContentType content_type,
1293 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001294 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001295 ++number_of_reconfigurations_;
1296 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1297 }
1298
Per Kjellanderdcef6412020-10-07 15:09:05 +02001299 void OnBitrateAllocationUpdated(
1300 const VideoBitrateAllocation& allocation) override {
1301 MutexLock lock(&mutex_);
1302 ++number_of_bitrate_allocations_;
1303 last_bitrate_allocation_ = allocation;
1304 }
1305
Per Kjellandera9434842020-10-15 17:53:22 +02001306 void OnVideoLayersAllocationUpdated(
1307 VideoLayersAllocation allocation) override {
1308 MutexLock lock(&mutex_);
1309 ++number_of_layers_allocations_;
1310 last_layers_allocation_ = allocation;
1311 rtc::StringBuilder log;
1312 for (const auto& layer : allocation.active_spatial_layers) {
1313 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1314 << "[";
1315 for (const auto target_bitrate :
1316 layer.target_bitrate_per_temporal_layer) {
1317 log << target_bitrate.kbps() << ",";
1318 }
1319 log << "]";
1320 }
1321 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1322 }
1323
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001324 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001325 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001326 TestEncoder* test_encoder_;
1327 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001328 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001329 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001330 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001331 uint32_t last_height_ = 0;
1332 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001333 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001334 size_t num_expected_layers_ = 1;
1335 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001336 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001337 int number_of_reconfigurations_ = 0;
1338 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001339 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1340 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001341 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1342 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001343 };
1344
Sergey Silkin5ee69672019-07-02 14:18:34 +02001345 class VideoBitrateAllocatorProxyFactory
1346 : public VideoBitrateAllocatorFactory {
1347 public:
1348 VideoBitrateAllocatorProxyFactory()
1349 : bitrate_allocator_factory_(
1350 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1351
1352 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1353 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001354 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001355 codec_config_ = codec;
1356 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1357 }
1358
1359 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001360 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001361 return codec_config_;
1362 }
1363
1364 private:
1365 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1366
Markus Handella3765182020-07-08 13:13:32 +02001367 mutable Mutex mutex_;
1368 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001369 };
1370
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001371 Clock* clock() { return time_controller_.GetClock(); }
1372 void AdvanceTime(TimeDelta duration) {
1373 time_controller_.AdvanceTime(duration);
1374 }
1375
1376 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1377
1378 protected:
1379 virtual TaskQueueFactory* GetTaskQueueFactory() {
1380 return time_controller_.GetTaskQueueFactory();
1381 }
1382
1383 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001384 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001385 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001386 int codec_width_;
1387 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001388 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001389 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001390 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001391 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001392 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001393 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001394 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001395 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001396};
1397
mflodmancc3d4422017-08-03 08:27:51 -07001398TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001400 DataRate::BitsPerSec(kTargetBitrateBps),
1401 DataRate::BitsPerSec(kTargetBitrateBps),
1402 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001403 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001404 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001405 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001406 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001407 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001408}
1409
mflodmancc3d4422017-08-03 08:27:51 -07001410TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001411 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001412 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001413 // The encoder will cache up to one frame for a short duration. Adding two
1414 // frames means that the first frame will be dropped and the second frame will
1415 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001416 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001417 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001418 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001419 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001420
Henrik Boström381d1092020-05-12 18:49:07 +02001421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001422 DataRate::BitsPerSec(kTargetBitrateBps),
1423 DataRate::BitsPerSec(kTargetBitrateBps),
1424 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001425
Sebastian Janssona3177052018-04-10 13:05:49 +02001426 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001427 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001428 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1429
1430 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001431 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001432}
1433
mflodmancc3d4422017-08-03 08:27:51 -07001434TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps),
1438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001439 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001440 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001441
Henrik Boström381d1092020-05-12 18:49:07 +02001442 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1443 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1444 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001445 // The encoder will cache up to one frame for a short duration. Adding two
1446 // frames means that the first frame will be dropped and the second frame will
1447 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001448 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001449 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001450
Henrik Boström381d1092020-05-12 18:49:07 +02001451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001452 DataRate::BitsPerSec(kTargetBitrateBps),
1453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001455 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001456 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1457 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001458 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001459}
1460
mflodmancc3d4422017-08-03 08:27:51 -07001461TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001463 DataRate::BitsPerSec(kTargetBitrateBps),
1464 DataRate::BitsPerSec(kTargetBitrateBps),
1465 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001466 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001467 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001468
1469 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001470 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001471
perkja49cbd32016-09-16 07:53:41 -07001472 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001473 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001474 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001475}
1476
mflodmancc3d4422017-08-03 08:27:51 -07001477TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001479 DataRate::BitsPerSec(kTargetBitrateBps),
1480 DataRate::BitsPerSec(kTargetBitrateBps),
1481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001482
perkja49cbd32016-09-16 07:53:41 -07001483 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001484 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001485
mflodmancc3d4422017-08-03 08:27:51 -07001486 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001487 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001488 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001489 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1490 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001491}
1492
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001493class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1494 public:
1495 VideoStreamEncoderBlockedTest() {}
1496
1497 TaskQueueFactory* GetTaskQueueFactory() override {
1498 return task_queue_factory_.get();
1499 }
1500
1501 private:
1502 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1503 CreateDefaultTaskQueueFactory();
1504};
1505
1506TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001508 DataRate::BitsPerSec(kTargetBitrateBps),
1509 DataRate::BitsPerSec(kTargetBitrateBps),
1510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001511
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001512 int dropped_count = 0;
1513 stats_proxy_->SetDroppedFrameCallback(
1514 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1515 ++dropped_count;
1516 });
1517
perkj26091b12016-09-01 01:17:40 -07001518 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001519 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001520 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001521 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1522 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001523 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1524 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001525 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001526 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001527
mflodmancc3d4422017-08-03 08:27:51 -07001528 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001529
1530 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001531}
1532
Noah Richards51db4212019-06-12 06:59:12 -07001533TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001538
1539 rtc::Event frame_destroyed_event;
1540 video_source_.IncomingCapturedFrame(
1541 CreateFakeNativeFrame(1, &frame_destroyed_event));
1542 ExpectDroppedFrame();
1543 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1544 video_stream_encoder_->Stop();
1545}
1546
1547TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1548 // Use the cropping factory.
1549 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001550 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001551 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1552 kMaxPayloadLength);
1553 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1554
1555 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001557 DataRate::BitsPerSec(kTargetBitrateBps),
1558 DataRate::BitsPerSec(kTargetBitrateBps),
1559 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001560 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1561 WaitForEncodedFrame(1);
1562 // The encoder will have been configured once.
1563 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1564 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1565 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1566
1567 // Now send in a fake frame that needs to be cropped as the width/height
1568 // aren't divisible by 4 (see CreateEncoderStreams above).
1569 rtc::Event frame_destroyed_event;
1570 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1571 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1572 ExpectDroppedFrame();
1573 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1574 video_stream_encoder_->Stop();
1575}
1576
Evan Shrubsole895556e2020-10-05 09:15:13 +02001577TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1579 DataRate::BitsPerSec(kTargetBitrateBps),
1580 DataRate::BitsPerSec(kTargetBitrateBps),
1581 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1582
1583 video_source_.IncomingCapturedFrame(
1584 CreateNV12Frame(1, codec_width_, codec_height_));
1585 WaitForEncodedFrame(1);
1586 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1587 fake_encoder_.GetLastInputPixelFormat());
1588 video_stream_encoder_->Stop();
1589}
1590
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001591TEST_F(VideoStreamEncoderTest,
1592 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1594 DataRate::BitsPerSec(kTargetBitrateBps),
1595 DataRate::BitsPerSec(kTargetBitrateBps),
1596 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1597
1598 fake_encoder_.SetPreferredPixelFormats({});
1599
1600 rtc::Event frame_destroyed_event;
1601 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1602 1, &frame_destroyed_event, codec_width_, codec_height_));
1603 WaitForEncodedFrame(1);
1604 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1605 fake_encoder_.GetLastInputPixelFormat());
1606 video_stream_encoder_->Stop();
1607}
1608
1609TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1611 DataRate::BitsPerSec(kTargetBitrateBps),
1612 DataRate::BitsPerSec(kTargetBitrateBps),
1613 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1614
1615 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1616
1617 rtc::Event frame_destroyed_event;
1618 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1619 1, &frame_destroyed_event, codec_width_, codec_height_));
1620 WaitForEncodedFrame(1);
1621 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1622 fake_encoder_.GetLastInputPixelFormat());
1623 video_stream_encoder_->Stop();
1624}
1625
1626TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1628 DataRate::BitsPerSec(kTargetBitrateBps),
1629 DataRate::BitsPerSec(kTargetBitrateBps),
1630 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1631
1632 // Fake NV12 native frame does not allow mapping to I444.
1633 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1634
1635 rtc::Event frame_destroyed_event;
1636 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1637 1, &frame_destroyed_event, codec_width_, codec_height_));
1638 WaitForEncodedFrame(1);
1639 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1640 fake_encoder_.GetLastInputPixelFormat());
1641 video_stream_encoder_->Stop();
1642}
1643
Evan Shrubsole895556e2020-10-05 09:15:13 +02001644TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1646 DataRate::BitsPerSec(kTargetBitrateBps),
1647 DataRate::BitsPerSec(kTargetBitrateBps),
1648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1649
1650 rtc::Event frame_destroyed_event;
1651 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1652 1, &frame_destroyed_event, codec_width_, codec_height_));
1653 WaitForEncodedFrame(1);
1654 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1655 fake_encoder_.GetLastInputPixelFormat());
1656 video_stream_encoder_->Stop();
1657}
1658
Ying Wang9b881ab2020-02-07 14:29:32 +01001659TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps),
1663 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001664 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1665 WaitForEncodedFrame(1);
1666
Henrik Boström381d1092020-05-12 18:49:07 +02001667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001668 DataRate::BitsPerSec(kTargetBitrateBps),
1669 DataRate::BitsPerSec(kTargetBitrateBps),
1670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001671 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1672 // frames. Adding two frames means that the first frame will be dropped and
1673 // the second frame will be sent to the encoder.
1674 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1675 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1676 WaitForEncodedFrame(3);
1677 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1678 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1679 WaitForEncodedFrame(5);
1680 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1681 video_stream_encoder_->Stop();
1682}
1683
mflodmancc3d4422017-08-03 08:27:51 -07001684TEST_F(VideoStreamEncoderTest,
1685 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps),
1689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001690 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001691
1692 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001693 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001694 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001695 // The encoder will have been configured once when the first frame is
1696 // received.
1697 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001698
1699 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001700 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001701 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001702 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001703 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001704
1705 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001706 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001707 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001708 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001709 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001710
mflodmancc3d4422017-08-03 08:27:51 -07001711 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001712}
1713
mflodmancc3d4422017-08-03 08:27:51 -07001714TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001716 DataRate::BitsPerSec(kTargetBitrateBps),
1717 DataRate::BitsPerSec(kTargetBitrateBps),
1718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001719
1720 // Capture a frame and wait for it to synchronize with the encoder thread.
1721 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001722 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001723 // The encoder will have been configured once.
1724 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001725 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1726 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1727
1728 codec_width_ *= 2;
1729 codec_height_ *= 2;
1730 // Capture a frame with a higher resolution and wait for it to synchronize
1731 // with the encoder thread.
1732 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001733 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001734 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1735 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001736 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001737
mflodmancc3d4422017-08-03 08:27:51 -07001738 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001739}
1740
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001741TEST_F(VideoStreamEncoderTest,
1742 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001744 DataRate::BitsPerSec(kTargetBitrateBps),
1745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001747
1748 // Capture a frame and wait for it to synchronize with the encoder thread.
1749 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1750 WaitForEncodedFrame(1);
1751
1752 VideoEncoderConfig video_encoder_config;
1753 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1754 // Changing the max payload data length recreates encoder.
1755 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1756 kMaxPayloadLength / 2);
1757
1758 // Capture a frame and wait for it to synchronize with the encoder thread.
1759 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1760 WaitForEncodedFrame(2);
1761 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1762
1763 video_stream_encoder_->Stop();
1764}
1765
Sergey Silkin5ee69672019-07-02 14:18:34 +02001766TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001768 DataRate::BitsPerSec(kTargetBitrateBps),
1769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001771
1772 VideoEncoderConfig video_encoder_config;
1773 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1774 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1775 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1776 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1777 kMaxPayloadLength);
1778
1779 // Capture a frame and wait for it to synchronize with the encoder thread.
1780 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1781 WaitForEncodedFrame(1);
1782 // The encoder will have been configured once when the first frame is
1783 // received.
1784 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1785 EXPECT_EQ(kTargetBitrateBps,
1786 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1787 EXPECT_EQ(kStartBitrateBps,
1788 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1789
Sergey Silkin6456e352019-07-08 17:56:40 +02001790 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1791 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001792 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1793 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1794 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1795 kMaxPayloadLength);
1796
1797 // Capture a frame and wait for it to synchronize with the encoder thread.
1798 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1799 WaitForEncodedFrame(2);
1800 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1801 // Bitrate limits have changed - rate allocator should be reconfigured,
1802 // encoder should not be reconfigured.
1803 EXPECT_EQ(kTargetBitrateBps * 2,
1804 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1805 EXPECT_EQ(kStartBitrateBps * 2,
1806 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1807 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1808
1809 video_stream_encoder_->Stop();
1810}
1811
Sergey Silkin6456e352019-07-08 17:56:40 +02001812TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001813 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001814 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001815 DataRate::BitsPerSec(kTargetBitrateBps),
1816 DataRate::BitsPerSec(kTargetBitrateBps),
1817 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001818
Sergey Silkincd02eba2020-01-20 14:48:40 +01001819 const uint32_t kMinEncBitrateKbps = 100;
1820 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001821 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001822 /*frame_size_pixels=*/codec_width_ * codec_height_,
1823 /*min_start_bitrate_bps=*/0,
1824 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1825 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001826 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1827
Sergey Silkincd02eba2020-01-20 14:48:40 +01001828 VideoEncoderConfig video_encoder_config;
1829 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1830 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1831 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1832 (kMinEncBitrateKbps + 1) * 1000;
1833 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1834 kMaxPayloadLength);
1835
1836 // When both encoder and app provide bitrate limits, the intersection of
1837 // provided sets should be used.
1838 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1839 WaitForEncodedFrame(1);
1840 EXPECT_EQ(kMaxEncBitrateKbps,
1841 bitrate_allocator_factory_.codec_config().maxBitrate);
1842 EXPECT_EQ(kMinEncBitrateKbps + 1,
1843 bitrate_allocator_factory_.codec_config().minBitrate);
1844
1845 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1846 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1847 (kMinEncBitrateKbps - 1) * 1000;
1848 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1849 kMaxPayloadLength);
1850 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001851 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001852 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001853 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001854 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001855 bitrate_allocator_factory_.codec_config().minBitrate);
1856
Sergey Silkincd02eba2020-01-20 14:48:40 +01001857 video_stream_encoder_->Stop();
1858}
1859
1860TEST_F(VideoStreamEncoderTest,
1861 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001863 DataRate::BitsPerSec(kTargetBitrateBps),
1864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001866
1867 const uint32_t kMinAppBitrateKbps = 100;
1868 const uint32_t kMaxAppBitrateKbps = 200;
1869 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1870 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1871 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1872 /*frame_size_pixels=*/codec_width_ * codec_height_,
1873 /*min_start_bitrate_bps=*/0,
1874 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1875 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1876 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1877
1878 VideoEncoderConfig video_encoder_config;
1879 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1880 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1881 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1882 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1884 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001885
Sergey Silkincd02eba2020-01-20 14:48:40 +01001886 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1887 WaitForEncodedFrame(1);
1888 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001889 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001890 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001891 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001892
1893 video_stream_encoder_->Stop();
1894}
1895
1896TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001897 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001899 DataRate::BitsPerSec(kTargetBitrateBps),
1900 DataRate::BitsPerSec(kTargetBitrateBps),
1901 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001902
1903 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001904 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001905 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001906 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001907 fake_encoder_.SetResolutionBitrateLimits(
1908 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1909
1910 VideoEncoderConfig video_encoder_config;
1911 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1912 video_encoder_config.max_bitrate_bps = 0;
1913 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1914 kMaxPayloadLength);
1915
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001916 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001917 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1918 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001919 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1920 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001921 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1922 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1923
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001924 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001925 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1926 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001927 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1928 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001929 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1930 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1931
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001932 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001933 // encoder for 360p should be used.
1934 video_source_.IncomingCapturedFrame(
1935 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1936 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001937 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1938 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001939 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1940 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1941
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001943 // ignored.
1944 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1945 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001946 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001948 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1949 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001950 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1951 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001952 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1953 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1954
1955 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1956 // for 270p should be used.
1957 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1958 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001959 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1960 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001961 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1962 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1963
1964 video_stream_encoder_->Stop();
1965}
1966
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001967TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001969 DataRate::BitsPerSec(kTargetBitrateBps),
1970 DataRate::BitsPerSec(kTargetBitrateBps),
1971 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001972
1973 VideoEncoderConfig video_encoder_config;
1974 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1975 video_encoder_config.max_bitrate_bps = 0;
1976 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1977 kMaxPayloadLength);
1978
1979 // Encode 720p frame to get the default encoder target bitrate.
1980 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1981 WaitForEncodedFrame(1);
1982 const uint32_t kDefaultTargetBitrateFor720pKbps =
1983 bitrate_allocator_factory_.codec_config()
1984 .simulcastStream[0]
1985 .targetBitrate;
1986
1987 // Set the max recommended encoder bitrate to something lower than the default
1988 // target bitrate.
1989 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1990 1280 * 720, 10 * 1000, 10 * 1000,
1991 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1992 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1993
1994 // Change resolution to trigger encoder reinitialization.
1995 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1996 WaitForEncodedFrame(2);
1997 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1998 WaitForEncodedFrame(3);
1999
2000 // Ensure the target bitrate is capped by the max bitrate.
2001 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2002 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2003 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2004 .simulcastStream[0]
2005 .targetBitrate *
2006 1000,
2007 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2008
2009 video_stream_encoder_->Stop();
2010}
2011
Åsa Perssona7e34d32021-01-20 15:36:13 +01002012TEST_F(VideoStreamEncoderTest,
2013 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2014 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2015 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2016 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2017 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2018 fake_encoder_.SetResolutionBitrateLimits(
2019 {kEncoderLimits270p, kEncoderLimits360p});
2020
2021 // Two streams, highest stream active.
2022 VideoEncoderConfig config;
2023 const int kNumStreams = 2;
2024 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2025 config.max_bitrate_bps = 0;
2026 config.simulcast_layers[0].active = false;
2027 config.simulcast_layers[1].active = true;
2028 config.video_stream_factory =
2029 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2030 "VP8", /*max qp*/ 56, /*screencast*/ false,
2031 /*screenshare enabled*/ false);
2032 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2033
2034 // The encoder bitrate limits for 270p should be used.
2035 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2036 EXPECT_FALSE(WaitForFrame(1000));
2037 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2038 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2039 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2040 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2041 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2042
2043 // The encoder bitrate limits for 360p should be used.
2044 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2045 EXPECT_FALSE(WaitForFrame(1000));
2046 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2047 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2048 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2049 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2050
2051 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2052 video_source_.IncomingCapturedFrame(
2053 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2054 EXPECT_FALSE(WaitForFrame(1000));
2055 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2056 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2057 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2058 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2059
2060 // Resolution higher than 360p. Encoder limits should be ignored.
2061 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2062 EXPECT_FALSE(WaitForFrame(1000));
2063 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2064 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2065 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2066 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2067 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2068 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2069 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2070 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2071
2072 // Resolution lower than 270p. The encoder limits for 270p should be used.
2073 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2074 EXPECT_FALSE(WaitForFrame(1000));
2075 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2076 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2077 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2078 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2079
2080 video_stream_encoder_->Stop();
2081}
2082
2083TEST_F(VideoStreamEncoderTest,
2084 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2085 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2086 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2087 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2088 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2089 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2090 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2091 fake_encoder_.SetResolutionBitrateLimits(
2092 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2093
2094 // Three streams, middle stream active.
2095 VideoEncoderConfig config;
2096 const int kNumStreams = 3;
2097 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2098 config.simulcast_layers[0].active = false;
2099 config.simulcast_layers[1].active = true;
2100 config.simulcast_layers[2].active = false;
2101 config.video_stream_factory =
2102 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2103 "VP8", /*max qp*/ 56, /*screencast*/ false,
2104 /*screenshare enabled*/ false);
2105 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2106
2107 // The encoder bitrate limits for 360p should be used.
2108 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2109 EXPECT_FALSE(WaitForFrame(1000));
2110 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2111 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2112 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2114 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2115
2116 // The encoder bitrate limits for 270p should be used.
2117 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2118 EXPECT_FALSE(WaitForFrame(1000));
2119 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2120 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2121 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2122 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2123
2124 video_stream_encoder_->Stop();
2125}
2126
2127TEST_F(VideoStreamEncoderTest,
2128 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2129 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2130 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2131 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2132 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2133 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2134 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2135 fake_encoder_.SetResolutionBitrateLimits(
2136 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2137
2138 // Three streams, lowest stream active.
2139 VideoEncoderConfig config;
2140 const int kNumStreams = 3;
2141 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2142 config.simulcast_layers[0].active = true;
2143 config.simulcast_layers[1].active = false;
2144 config.simulcast_layers[2].active = false;
2145 config.video_stream_factory =
2146 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2147 "VP8", /*max qp*/ 56, /*screencast*/ false,
2148 /*screenshare enabled*/ false);
2149 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2150
2151 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2152 // on lowest stream, limits for 270p should not be used
2153 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2154 EXPECT_FALSE(WaitForFrame(1000));
2155 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2156 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2157 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2158 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2159 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2160
2161 video_stream_encoder_->Stop();
2162}
2163
2164TEST_F(VideoStreamEncoderTest,
2165 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2166 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2167 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2168 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2169 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2170 fake_encoder_.SetResolutionBitrateLimits(
2171 {kEncoderLimits270p, kEncoderLimits360p});
2172 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2173
2174 // Two streams, highest stream active.
2175 VideoEncoderConfig config;
2176 const int kNumStreams = 2;
2177 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2178 config.simulcast_layers[0].active = false;
2179 config.simulcast_layers[1].active = true;
2180 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2181 config.video_stream_factory =
2182 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2183 "VP8", /*max qp*/ 56, /*screencast*/ false,
2184 /*screenshare enabled*/ false);
2185 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2186
2187 // The encoder bitrate limits for 270p should be used.
2188 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2191 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2192 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2193 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2194 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2195
2196 // The max configured bitrate is less than the encoder limit for 360p.
2197 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2198 EXPECT_FALSE(WaitForFrame(1000));
2199 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2200 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2201 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2202 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2203
2204 video_stream_encoder_->Stop();
2205}
2206
mflodmancc3d4422017-08-03 08:27:51 -07002207TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002208 EXPECT_TRUE(video_source_.has_sinks());
2209 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002210 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002211 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002212 EXPECT_FALSE(video_source_.has_sinks());
2213 EXPECT_TRUE(new_video_source.has_sinks());
2214
mflodmancc3d4422017-08-03 08:27:51 -07002215 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002216}
2217
mflodmancc3d4422017-08-03 08:27:51 -07002218TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002219 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002220 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002221 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002222 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002223}
2224
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002225class ResolutionAlignmentTest
2226 : public VideoStreamEncoderTest,
2227 public ::testing::WithParamInterface<
2228 ::testing::tuple<int, std::vector<double>>> {
2229 public:
2230 ResolutionAlignmentTest()
2231 : requested_alignment_(::testing::get<0>(GetParam())),
2232 scale_factors_(::testing::get<1>(GetParam())) {}
2233
2234 protected:
2235 const int requested_alignment_;
2236 const std::vector<double> scale_factors_;
2237};
2238
2239INSTANTIATE_TEST_SUITE_P(
2240 AlignmentAndScaleFactors,
2241 ResolutionAlignmentTest,
2242 ::testing::Combine(
2243 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2244 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2245 std::vector<double>{-1.0, -1.0},
2246 std::vector<double>{-1.0, -1.0, -1.0},
2247 std::vector<double>{4.0, 2.0, 1.0},
2248 std::vector<double>{9999.0, -1.0, 1.0},
2249 std::vector<double>{3.99, 2.01, 1.0},
2250 std::vector<double>{4.9, 1.7, 1.25},
2251 std::vector<double>{10.0, 4.0, 3.0},
2252 std::vector<double>{1.75, 3.5},
2253 std::vector<double>{1.5, 2.5},
2254 std::vector<double>{1.3, 1.0})));
2255
2256TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2257 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002258 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002259 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2260 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2261
2262 // Fill config with the scaling factor by which to reduce encoding size.
2263 const int num_streams = scale_factors_.size();
2264 VideoEncoderConfig config;
2265 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2266 for (int i = 0; i < num_streams; ++i) {
2267 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2268 }
2269 config.video_stream_factory =
2270 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2271 "VP8", /*max qp*/ 56, /*screencast*/ false,
2272 /*screenshare enabled*/ false);
2273 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2274
Henrik Boström381d1092020-05-12 18:49:07 +02002275 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002276 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2277 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2278 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2279 // Wait for all layers before triggering event.
2280 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002281
2282 // On the 1st frame, we should have initialized the encoder and
2283 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002284 int64_t timestamp_ms = kFrameIntervalMs;
2285 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2286 WaitForEncodedFrame(timestamp_ms);
2287 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002288
2289 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2290 // (It's up the to the encoder to potentially drop the previous frame,
2291 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002292 timestamp_ms += kFrameIntervalMs;
2293 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2294 WaitForEncodedFrame(timestamp_ms);
2295 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2296
2297 VideoCodec codec = fake_encoder_.video_codec();
2298 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2299 // Frame size should be a multiple of the requested alignment.
2300 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2301 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2302 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2303 // Aspect ratio should match.
2304 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2305 codec.height * codec.simulcastStream[i].width);
2306 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002307
2308 video_stream_encoder_->Stop();
2309}
2310
Jonathan Yubc771b72017-12-08 17:04:29 -08002311TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2312 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002313 const int kWidth = 1280;
2314 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002315
2316 // We rely on the automatic resolution adaptation, but we handle framerate
2317 // adaptation manually by mocking the stats proxy.
2318 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002319
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002320 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002322 DataRate::BitsPerSec(kTargetBitrateBps),
2323 DataRate::BitsPerSec(kTargetBitrateBps),
2324 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002325 video_stream_encoder_->SetSource(&video_source_,
2326 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002327 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002328 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002329 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002330 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2331
Jonathan Yubc771b72017-12-08 17:04:29 -08002332 // Adapt down as far as possible.
2333 rtc::VideoSinkWants last_wants;
2334 int64_t t = 1;
2335 int loop_count = 0;
2336 do {
2337 ++loop_count;
2338 last_wants = video_source_.sink_wants();
2339
2340 // Simulate the framerate we've been asked to adapt to.
2341 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2342 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2343 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2344 mock_stats.input_frame_rate = fps;
2345 stats_proxy_->SetMockStats(mock_stats);
2346
2347 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2348 sink_.WaitForEncodedFrame(t);
2349 t += frame_interval_ms;
2350
mflodmancc3d4422017-08-03 08:27:51 -07002351 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002352 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002353 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002354 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2355 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002356 } while (video_source_.sink_wants().max_pixel_count <
2357 last_wants.max_pixel_count ||
2358 video_source_.sink_wants().max_framerate_fps <
2359 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002360
Jonathan Yubc771b72017-12-08 17:04:29 -08002361 // Verify that we've adapted all the way down.
2362 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002363 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002364 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2365 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002366 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002367 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2368 *video_source_.last_sent_height());
2369 EXPECT_EQ(kMinBalancedFramerateFps,
2370 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002371
Jonathan Yubc771b72017-12-08 17:04:29 -08002372 // Adapt back up the same number of times we adapted down.
2373 for (int i = 0; i < loop_count - 1; ++i) {
2374 last_wants = video_source_.sink_wants();
2375
2376 // Simulate the framerate we've been asked to adapt to.
2377 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2378 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2379 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2380 mock_stats.input_frame_rate = fps;
2381 stats_proxy_->SetMockStats(mock_stats);
2382
2383 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2384 sink_.WaitForEncodedFrame(t);
2385 t += frame_interval_ms;
2386
Henrik Boström91aa7322020-04-28 12:24:33 +02002387 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002388 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002389 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002390 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2391 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002392 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2393 last_wants.max_pixel_count ||
2394 video_source_.sink_wants().max_framerate_fps >
2395 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002396 }
2397
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002398 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002399 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002400 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002401 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2402 EXPECT_EQ((loop_count - 1) * 2,
2403 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002404
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002406}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002407
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002408TEST_F(VideoStreamEncoderTest,
2409 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2410 video_stream_encoder_->OnBitrateUpdated(
2411 DataRate::BitsPerSec(kTargetBitrateBps),
2412 DataRate::BitsPerSec(kTargetBitrateBps),
2413 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002414 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002415
2416 const int kFrameWidth = 1280;
2417 const int kFrameHeight = 720;
2418
2419 int64_t ntp_time = kFrameIntervalMs;
2420
2421 // Force an input frame rate to be available, or the adaptation call won't
2422 // know what framerate to adapt form.
2423 const int kInputFps = 30;
2424 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2425 stats.input_frame_rate = kInputFps;
2426 stats_proxy_->SetMockStats(stats);
2427
2428 video_source_.set_adaptation_enabled(true);
2429 video_stream_encoder_->SetSource(
2430 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002431 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002432 video_source_.IncomingCapturedFrame(
2433 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2434 sink_.WaitForEncodedFrame(ntp_time);
2435 ntp_time += kFrameIntervalMs;
2436
2437 // Trigger CPU overuse.
2438 video_stream_encoder_->TriggerCpuOveruse();
2439 video_source_.IncomingCapturedFrame(
2440 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2441 sink_.WaitForEncodedFrame(ntp_time);
2442 ntp_time += kFrameIntervalMs;
2443
2444 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2445 EXPECT_EQ(std::numeric_limits<int>::max(),
2446 video_source_.sink_wants().max_pixel_count);
2447 // Some framerate constraint should be set.
2448 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2449 EXPECT_LT(restricted_fps, kInputFps);
2450 video_source_.IncomingCapturedFrame(
2451 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2452 sink_.WaitForEncodedFrame(ntp_time);
2453 ntp_time += 100;
2454
Henrik Boström2671dac2020-05-19 16:29:09 +02002455 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002456 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2457 // Give the encoder queue time to process the change in degradation preference
2458 // by waiting for an encoded frame.
2459 video_source_.IncomingCapturedFrame(
2460 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2461 sink_.WaitForEncodedFrame(ntp_time);
2462 ntp_time += kFrameIntervalMs;
2463
2464 video_stream_encoder_->TriggerQualityLow();
2465 video_source_.IncomingCapturedFrame(
2466 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2467 sink_.WaitForEncodedFrame(ntp_time);
2468 ntp_time += kFrameIntervalMs;
2469
2470 // Some resolution constraint should be set.
2471 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2472 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2473 kFrameWidth * kFrameHeight);
2474 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2475
2476 int pixel_count = video_source_.sink_wants().max_pixel_count;
2477 // Triggering a CPU underuse should not change the sink wants since it has
2478 // not been overused for resolution since we changed degradation preference.
2479 video_stream_encoder_->TriggerCpuUnderuse();
2480 video_source_.IncomingCapturedFrame(
2481 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2482 sink_.WaitForEncodedFrame(ntp_time);
2483 ntp_time += kFrameIntervalMs;
2484 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2485 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2486
Evan Shrubsole64469032020-06-11 10:45:29 +02002487 // Change the degradation preference back. CPU underuse should not adapt since
2488 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002489 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002490 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2491 video_source_.IncomingCapturedFrame(
2492 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2493 sink_.WaitForEncodedFrame(ntp_time);
2494 ntp_time += 100;
2495 // Resolution adaptations is gone after changing degradation preference.
2496 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2497 EXPECT_EQ(std::numeric_limits<int>::max(),
2498 video_source_.sink_wants().max_pixel_count);
2499 // The fps adaptation from above is now back.
2500 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2501
2502 // Trigger CPU underuse.
2503 video_stream_encoder_->TriggerCpuUnderuse();
2504 video_source_.IncomingCapturedFrame(
2505 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2506 sink_.WaitForEncodedFrame(ntp_time);
2507 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002508 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2509
2510 // Trigger QP underuse, fps should return to normal.
2511 video_stream_encoder_->TriggerQualityHigh();
2512 video_source_.IncomingCapturedFrame(
2513 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2514 sink_.WaitForEncodedFrame(ntp_time);
2515 ntp_time += kFrameIntervalMs;
2516 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002517
2518 video_stream_encoder_->Stop();
2519}
2520
mflodmancc3d4422017-08-03 08:27:51 -07002521TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002523 DataRate::BitsPerSec(kTargetBitrateBps),
2524 DataRate::BitsPerSec(kTargetBitrateBps),
2525 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002526 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002527
sprangc5d62e22017-04-02 23:53:04 -07002528 const int kFrameWidth = 1280;
2529 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002530
Åsa Persson8c1bf952018-09-13 10:42:19 +02002531 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002532
kthelgason5e13d412016-12-01 03:59:51 -08002533 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002534 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002535 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002536 frame_timestamp += kFrameIntervalMs;
2537
perkj803d97f2016-11-01 11:45:46 -07002538 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002539 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002540 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002541 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002542 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002543 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002544
asapersson0944a802017-04-07 00:57:58 -07002545 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002546 // wanted resolution.
2547 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2548 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2549 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002550 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002551
2552 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002553 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002554 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002555 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002556 // Give the encoder queue time to process the change in degradation preference
2557 // by waiting for an encoded frame.
2558 new_video_source.IncomingCapturedFrame(
2559 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2560 sink_.WaitForEncodedFrame(frame_timestamp);
2561 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002562 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002563 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002564
sprangc5d62e22017-04-02 23:53:04 -07002565 // Force an input frame rate to be available, or the adaptation call won't
2566 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002567 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002568 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002569 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002570 stats_proxy_->SetMockStats(stats);
2571
mflodmancc3d4422017-08-03 08:27:51 -07002572 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002573 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002574 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002575 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002576 frame_timestamp += kFrameIntervalMs;
2577
2578 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002579 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002580 EXPECT_EQ(std::numeric_limits<int>::max(),
2581 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002582 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002583
asapersson02465b82017-04-10 01:12:52 -07002584 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002585 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2586 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002587 // Give the encoder queue time to process the change in degradation preference
2588 // by waiting for an encoded frame.
2589 new_video_source.IncomingCapturedFrame(
2590 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2591 sink_.WaitForEncodedFrame(frame_timestamp);
2592 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002593 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002594
mflodmancc3d4422017-08-03 08:27:51 -07002595 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002596 new_video_source.IncomingCapturedFrame(
2597 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002598 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002599 frame_timestamp += kFrameIntervalMs;
2600
2601 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002602 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002603
2604 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002605 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002606 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002607 // Give the encoder queue time to process the change in degradation preference
2608 // by waiting for an encoded frame.
2609 new_video_source.IncomingCapturedFrame(
2610 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2611 sink_.WaitForEncodedFrame(frame_timestamp);
2612 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002613 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2614 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002615 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002616 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002617
2618 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002619 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002620 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002621 // Give the encoder queue time to process the change in degradation preference
2622 // by waiting for an encoded frame.
2623 new_video_source.IncomingCapturedFrame(
2624 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2625 sink_.WaitForEncodedFrame(frame_timestamp);
2626 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002627 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2628 EXPECT_EQ(std::numeric_limits<int>::max(),
2629 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002630 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002631
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002633}
2634
mflodmancc3d4422017-08-03 08:27:51 -07002635TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002637 DataRate::BitsPerSec(kTargetBitrateBps),
2638 DataRate::BitsPerSec(kTargetBitrateBps),
2639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002640
asaperssonfab67072017-04-04 05:51:49 -07002641 const int kWidth = 1280;
2642 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002643 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002644 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002645 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2646 EXPECT_FALSE(stats.bw_limited_resolution);
2647 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2648
2649 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002650 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002651 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002652 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002653
2654 stats = stats_proxy_->GetStats();
2655 EXPECT_TRUE(stats.bw_limited_resolution);
2656 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2657
2658 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002659 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002660 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002661 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002662
2663 stats = stats_proxy_->GetStats();
2664 EXPECT_FALSE(stats.bw_limited_resolution);
2665 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2666 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2667
mflodmancc3d4422017-08-03 08:27:51 -07002668 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002669}
2670
mflodmancc3d4422017-08-03 08:27:51 -07002671TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002673 DataRate::BitsPerSec(kTargetBitrateBps),
2674 DataRate::BitsPerSec(kTargetBitrateBps),
2675 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002676
2677 const int kWidth = 1280;
2678 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002679 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002680 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002681 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2682 EXPECT_FALSE(stats.cpu_limited_resolution);
2683 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2684
2685 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002686 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002687 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002688 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002689
2690 stats = stats_proxy_->GetStats();
2691 EXPECT_TRUE(stats.cpu_limited_resolution);
2692 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2693
2694 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002695 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002696 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002697 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002698
2699 stats = stats_proxy_->GetStats();
2700 EXPECT_FALSE(stats.cpu_limited_resolution);
2701 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002702 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002703
mflodmancc3d4422017-08-03 08:27:51 -07002704 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002705}
2706
mflodmancc3d4422017-08-03 08:27:51 -07002707TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002709 DataRate::BitsPerSec(kTargetBitrateBps),
2710 DataRate::BitsPerSec(kTargetBitrateBps),
2711 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002712
asaperssonfab67072017-04-04 05:51:49 -07002713 const int kWidth = 1280;
2714 const int kHeight = 720;
2715 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002716 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002717 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002718 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002719 EXPECT_FALSE(stats.cpu_limited_resolution);
2720 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2721
asaperssonfab67072017-04-04 05:51:49 -07002722 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002723 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002724 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002725 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002726 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002727 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002728 EXPECT_TRUE(stats.cpu_limited_resolution);
2729 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2730
2731 // Set new source with adaptation still enabled.
2732 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002733 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002734 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002735
asaperssonfab67072017-04-04 05:51:49 -07002736 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002737 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002738 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002739 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002740 EXPECT_TRUE(stats.cpu_limited_resolution);
2741 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2742
2743 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002744 video_stream_encoder_->SetSource(&new_video_source,
2745 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002746
asaperssonfab67072017-04-04 05:51:49 -07002747 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002748 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002749 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002750 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002751 EXPECT_FALSE(stats.cpu_limited_resolution);
2752 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2753
2754 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002755 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002756 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002757
asaperssonfab67072017-04-04 05:51:49 -07002758 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002759 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002760 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002761 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002762 EXPECT_TRUE(stats.cpu_limited_resolution);
2763 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2764
asaperssonfab67072017-04-04 05:51:49 -07002765 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002766 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002767 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002768 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002769 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002770 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002771 EXPECT_FALSE(stats.cpu_limited_resolution);
2772 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002773 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002774
mflodmancc3d4422017-08-03 08:27:51 -07002775 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002776}
2777
mflodmancc3d4422017-08-03 08:27:51 -07002778TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002780 DataRate::BitsPerSec(kTargetBitrateBps),
2781 DataRate::BitsPerSec(kTargetBitrateBps),
2782 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002783
asaperssonfab67072017-04-04 05:51:49 -07002784 const int kWidth = 1280;
2785 const int kHeight = 720;
2786 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002787 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002788 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002789 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002790 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002791 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002792
2793 // Set new source with adaptation still enabled.
2794 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002795 video_stream_encoder_->SetSource(&new_video_source,
2796 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002797
asaperssonfab67072017-04-04 05:51:49 -07002798 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002799 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002800 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002801 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002802 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002803 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002804
asaperssonfab67072017-04-04 05:51:49 -07002805 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002807 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002808 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002809 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002810 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002811 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002812 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002813
asaperssonfab67072017-04-04 05:51:49 -07002814 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002815 video_stream_encoder_->SetSource(&new_video_source,
2816 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002817
asaperssonfab67072017-04-04 05:51:49 -07002818 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002819 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002820 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002821 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002822 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002823 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002824
asapersson02465b82017-04-10 01:12:52 -07002825 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002826 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002827 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002828
asaperssonfab67072017-04-04 05:51:49 -07002829 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002830 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002831 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002832 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002833 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002834 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2835 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002836
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002838}
2839
mflodmancc3d4422017-08-03 08:27:51 -07002840TEST_F(VideoStreamEncoderTest,
2841 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002842 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002843 DataRate::BitsPerSec(kTargetBitrateBps),
2844 DataRate::BitsPerSec(kTargetBitrateBps),
2845 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002846
2847 const int kWidth = 1280;
2848 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002849 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002850 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002851 video_source_.IncomingCapturedFrame(
2852 CreateFrame(timestamp_ms, kWidth, kHeight));
2853 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2857
2858 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002860 timestamp_ms += kFrameIntervalMs;
2861 video_source_.IncomingCapturedFrame(
2862 CreateFrame(timestamp_ms, kWidth, kHeight));
2863 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002864 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2865 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2866 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2867
2868 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002870 timestamp_ms += kFrameIntervalMs;
2871 video_source_.IncomingCapturedFrame(
2872 CreateFrame(timestamp_ms, kWidth, kHeight));
2873 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002874 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2875 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2876 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2877
Niels Möller4db138e2018-04-19 09:04:13 +02002878 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002879 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002880
2881 VideoEncoderConfig video_encoder_config;
2882 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2883 // Make format different, to force recreation of encoder.
2884 video_encoder_config.video_format.parameters["foo"] = "foo";
2885 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002886 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002887 timestamp_ms += kFrameIntervalMs;
2888 video_source_.IncomingCapturedFrame(
2889 CreateFrame(timestamp_ms, kWidth, kHeight));
2890 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002891 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2892 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2894
mflodmancc3d4422017-08-03 08:27:51 -07002895 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002896}
2897
mflodmancc3d4422017-08-03 08:27:51 -07002898TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002899 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002900 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002901 DataRate::BitsPerSec(kTargetBitrateBps),
2902 DataRate::BitsPerSec(kTargetBitrateBps),
2903 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2904
2905 const int kWidth = 1280;
2906 const int kHeight = 720;
2907 int sequence = 1;
2908
2909 // Enable BALANCED preference, no initial limitation.
2910 test::FrameForwarder source;
2911 video_stream_encoder_->SetSource(&source,
2912 webrtc::DegradationPreference::BALANCED);
2913 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2914 WaitForEncodedFrame(sequence++);
2915 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2916 EXPECT_FALSE(stats.cpu_limited_resolution);
2917 EXPECT_FALSE(stats.cpu_limited_framerate);
2918 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2919
2920 // Trigger CPU overuse, should now adapt down.
2921 video_stream_encoder_->TriggerCpuOveruse();
2922 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2923 WaitForEncodedFrame(sequence++);
2924 stats = stats_proxy_->GetStats();
2925 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2926
2927 // Set new degradation preference should clear restrictions since we changed
2928 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002929 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002930 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2931 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2932 WaitForEncodedFrame(sequence++);
2933 stats = stats_proxy_->GetStats();
2934 EXPECT_FALSE(stats.cpu_limited_resolution);
2935 EXPECT_FALSE(stats.cpu_limited_framerate);
2936 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2937
2938 // Force an input frame rate to be available, or the adaptation call won't
2939 // know what framerate to adapt from.
2940 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2941 mock_stats.input_frame_rate = 30;
2942 stats_proxy_->SetMockStats(mock_stats);
2943 video_stream_encoder_->TriggerCpuOveruse();
2944 stats_proxy_->ResetMockStats();
2945 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2946 WaitForEncodedFrame(sequence++);
2947
2948 // We have now adapted once.
2949 stats = stats_proxy_->GetStats();
2950 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2951
2952 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002953 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2954 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002955 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2956 WaitForEncodedFrame(sequence++);
2957 stats = stats_proxy_->GetStats();
2958 EXPECT_FALSE(stats.cpu_limited_resolution);
2959 EXPECT_FALSE(stats.cpu_limited_framerate);
2960 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2961
2962 video_stream_encoder_->Stop();
2963}
2964
2965TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002966 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002968 DataRate::BitsPerSec(kTargetBitrateBps),
2969 DataRate::BitsPerSec(kTargetBitrateBps),
2970 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002971
asapersson0944a802017-04-07 00:57:58 -07002972 const int kWidth = 1280;
2973 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002974 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002975
asaperssonfab67072017-04-04 05:51:49 -07002976 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002977 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002978 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002979 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002980 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002981 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2982
asapersson02465b82017-04-10 01:12:52 -07002983 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002985 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002986 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002987 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002988 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002989 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002990 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2991
2992 // Set new source with adaptation still enabled.
2993 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002994 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002995 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002996
2997 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002998 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002999 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003000 stats = stats_proxy_->GetStats();
3001 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003002 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003003 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3004
sprangc5d62e22017-04-02 23:53:04 -07003005 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003007 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003008 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003009 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003010 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003011 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003012 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003013 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003014 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003015 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3016
sprangc5d62e22017-04-02 23:53:04 -07003017 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003018 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003019 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3020 mock_stats.input_frame_rate = 30;
3021 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003022 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003023 stats_proxy_->ResetMockStats();
3024
3025 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003026 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003027 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003028
3029 // Framerate now adapted.
3030 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003031 EXPECT_FALSE(stats.cpu_limited_resolution);
3032 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003033 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3034
3035 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003036 video_stream_encoder_->SetSource(&new_video_source,
3037 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003038 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003039 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003040 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003041
3042 stats = stats_proxy_->GetStats();
3043 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003044 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003045 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3046
3047 // Try to trigger overuse. Should not succeed.
3048 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003049 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003050 stats_proxy_->ResetMockStats();
3051
3052 stats = stats_proxy_->GetStats();
3053 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003054 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003055 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3056
3057 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003058 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003059 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003060 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003061 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003062 stats = stats_proxy_->GetStats();
3063 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003064 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003065 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003066
3067 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003068 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003069 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003070 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003071 stats = stats_proxy_->GetStats();
3072 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003073 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003074 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3075
3076 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003077 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003078 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003079 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003080 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003081 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003082 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003083 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003084 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003085 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003086 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3087
3088 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003089 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003090 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003091 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003092 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003093 stats = stats_proxy_->GetStats();
3094 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003095 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003096 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003097 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003098
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003100}
3101
mflodmancc3d4422017-08-03 08:27:51 -07003102TEST_F(VideoStreamEncoderTest,
3103 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003104 const int kWidth = 1280;
3105 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003107 DataRate::BitsPerSec(kTargetBitrateBps),
3108 DataRate::BitsPerSec(kTargetBitrateBps),
3109 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003110
asaperssonfab67072017-04-04 05:51:49 -07003111 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003112 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003113
asaperssonfab67072017-04-04 05:51:49 -07003114 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003115 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003116
asaperssonfab67072017-04-04 05:51:49 -07003117 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003118 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003119
asaperssonfab67072017-04-04 05:51:49 -07003120 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003121 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003122
kthelgason876222f2016-11-29 01:44:11 -08003123 // Expect a scale down.
3124 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003125 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003126
asapersson02465b82017-04-10 01:12:52 -07003127 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003128 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003129 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003130 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003131
asaperssonfab67072017-04-04 05:51:49 -07003132 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003133 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003134 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003135 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003136
asaperssonfab67072017-04-04 05:51:49 -07003137 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003138 EXPECT_EQ(std::numeric_limits<int>::max(),
3139 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003140
asaperssonfab67072017-04-04 05:51:49 -07003141 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003142 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003143 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003144 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003145
asapersson02465b82017-04-10 01:12:52 -07003146 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003147 EXPECT_EQ(std::numeric_limits<int>::max(),
3148 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003149
mflodmancc3d4422017-08-03 08:27:51 -07003150 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003151}
3152
mflodmancc3d4422017-08-03 08:27:51 -07003153TEST_F(VideoStreamEncoderTest,
3154 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003155 const int kWidth = 1280;
3156 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003157 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003158 DataRate::BitsPerSec(kTargetBitrateBps),
3159 DataRate::BitsPerSec(kTargetBitrateBps),
3160 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003161
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003162 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003163 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003165 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003166
3167 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003168 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003169 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003170 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3171 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3172
3173 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003174 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003175 EXPECT_THAT(source.sink_wants(),
3176 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003177 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3178 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3179 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3180
3181 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003182 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003183 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3184 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3185 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3186
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003188}
3189
mflodmancc3d4422017-08-03 08:27:51 -07003190TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003191 const int kWidth = 1280;
3192 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003194 DataRate::BitsPerSec(kTargetBitrateBps),
3195 DataRate::BitsPerSec(kTargetBitrateBps),
3196 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003197
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003198 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003199 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003200 video_stream_encoder_->SetSource(&source,
3201 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003202 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3203 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003204 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003205
3206 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003208 EXPECT_THAT(source.sink_wants(),
3209 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003210 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3211 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3212 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3213
3214 // Trigger adapt down for same input resolution, expect no change.
3215 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3216 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003217 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003218 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3219 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3220 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3221
3222 // Trigger adapt down for larger input resolution, expect no change.
3223 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3224 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003225 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003226 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3227 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3228 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3229
mflodmancc3d4422017-08-03 08:27:51 -07003230 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003231}
3232
mflodmancc3d4422017-08-03 08:27:51 -07003233TEST_F(VideoStreamEncoderTest,
3234 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003235 const int kWidth = 1280;
3236 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003238 DataRate::BitsPerSec(kTargetBitrateBps),
3239 DataRate::BitsPerSec(kTargetBitrateBps),
3240 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003241
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003242 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003243 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003244 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003245 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003246
3247 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003249 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003250 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3251 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3252
3253 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003254 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003255 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003256 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3257 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3258
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003260}
3261
mflodmancc3d4422017-08-03 08:27:51 -07003262TEST_F(VideoStreamEncoderTest,
3263 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003264 const int kWidth = 1280;
3265 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003267 DataRate::BitsPerSec(kTargetBitrateBps),
3268 DataRate::BitsPerSec(kTargetBitrateBps),
3269 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003270
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003271 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003272 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003273 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003274 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003275
3276 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003277 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003278 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003279 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003280 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3281
3282 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003283 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003284 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003285 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003286 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3287
mflodmancc3d4422017-08-03 08:27:51 -07003288 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003289}
3290
mflodmancc3d4422017-08-03 08:27:51 -07003291TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003292 const int kWidth = 1280;
3293 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003294 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003295 DataRate::BitsPerSec(kTargetBitrateBps),
3296 DataRate::BitsPerSec(kTargetBitrateBps),
3297 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003298
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003299 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003300 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003301 video_stream_encoder_->SetSource(&source,
3302 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003303
3304 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3305 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003306 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3308 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3309 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3310
3311 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003312 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003313 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3315 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3316 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3317
mflodmancc3d4422017-08-03 08:27:51 -07003318 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003319}
3320
mflodmancc3d4422017-08-03 08:27:51 -07003321TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003322 const int kWidth = 1280;
3323 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003325 DataRate::BitsPerSec(kTargetBitrateBps),
3326 DataRate::BitsPerSec(kTargetBitrateBps),
3327 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003328
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003329 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003330 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003331 video_stream_encoder_->SetSource(&source,
3332 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003333
3334 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3335 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003336 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003337 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3338 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3339 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3340
3341 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003342 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003343 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3345 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3347
mflodmancc3d4422017-08-03 08:27:51 -07003348 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003349}
3350
mflodmancc3d4422017-08-03 08:27:51 -07003351TEST_F(VideoStreamEncoderTest,
3352 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003353 const int kWidth = 1280;
3354 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003356 DataRate::BitsPerSec(kTargetBitrateBps),
3357 DataRate::BitsPerSec(kTargetBitrateBps),
3358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003359
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003360 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003361 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003362 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003363 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003364 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003365
3366 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003367 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003368 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003369 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3370 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3371
3372 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003373 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003374 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003375 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003376 EXPECT_THAT(source.sink_wants(),
3377 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003378 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3379 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3380
3381 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003383 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3385 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3386 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3387
mflodmancc3d4422017-08-03 08:27:51 -07003388 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003389}
3390
mflodmancc3d4422017-08-03 08:27:51 -07003391TEST_F(VideoStreamEncoderTest,
3392 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003393 const int kWidth = 1280;
3394 const int kHeight = 720;
3395 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003397 DataRate::BitsPerSec(kTargetBitrateBps),
3398 DataRate::BitsPerSec(kTargetBitrateBps),
3399 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003400
3401 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3402 stats.input_frame_rate = kInputFps;
3403 stats_proxy_->SetMockStats(stats);
3404
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003405 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003406 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3407 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003408 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003409
3410 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003411 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003412 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3413 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003414 EXPECT_THAT(video_source_.sink_wants(),
3415 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003416
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003417 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003418 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003419 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003420 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003421 // Give the encoder queue time to process the change in degradation preference
3422 // by waiting for an encoded frame.
3423 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3424 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003425 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003426
3427 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003428 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003429 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3430 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003431 EXPECT_THAT(new_video_source.sink_wants(),
3432 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003433
3434 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003435 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003436 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003437
mflodmancc3d4422017-08-03 08:27:51 -07003438 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003439}
3440
mflodmancc3d4422017-08-03 08:27:51 -07003441TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003442 const int kWidth = 1280;
3443 const int kHeight = 720;
3444 const size_t kNumFrames = 10;
3445
Henrik Boström381d1092020-05-12 18:49:07 +02003446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003447 DataRate::BitsPerSec(kTargetBitrateBps),
3448 DataRate::BitsPerSec(kTargetBitrateBps),
3449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003450
asaperssond0de2952017-04-21 01:47:31 -07003451 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003452 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003453 video_source_.set_adaptation_enabled(true);
3454
3455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3457
3458 int downscales = 0;
3459 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003460 video_source_.IncomingCapturedFrame(
3461 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3462 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003463
asaperssonfab67072017-04-04 05:51:49 -07003464 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003465 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003466 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003467 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003468
3469 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3470 ++downscales;
3471
3472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3473 EXPECT_EQ(downscales,
3474 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003476 }
mflodmancc3d4422017-08-03 08:27:51 -07003477 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003478}
3479
mflodmancc3d4422017-08-03 08:27:51 -07003480TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003481 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3482 const int kWidth = 1280;
3483 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003485 DataRate::BitsPerSec(kTargetBitrateBps),
3486 DataRate::BitsPerSec(kTargetBitrateBps),
3487 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003488
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003489 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003490 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003491 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003492 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003493 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003494
Åsa Persson8c1bf952018-09-13 10:42:19 +02003495 int64_t timestamp_ms = kFrameIntervalMs;
3496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003497 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003499 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3500 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3501
3502 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003503 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003504 timestamp_ms += kFrameIntervalMs;
3505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3506 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003507 EXPECT_THAT(source.sink_wants(),
3508 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003509 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3510 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3511
3512 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003513 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003514 timestamp_ms += kFrameIntervalMs;
3515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003516 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003517 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003518 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3519 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3520
3521 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003522 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003523 timestamp_ms += kFrameIntervalMs;
3524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3525 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003526 EXPECT_THAT(source.sink_wants(),
3527 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003528 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3529 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3530
3531 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003532 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003533 timestamp_ms += kFrameIntervalMs;
3534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003535 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003536 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003537 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3538 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3539
mflodmancc3d4422017-08-03 08:27:51 -07003540 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003541}
3542
mflodmancc3d4422017-08-03 08:27:51 -07003543TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003544 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3545 const int kWidth = 1280;
3546 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003548 DataRate::BitsPerSec(kTargetBitrateBps),
3549 DataRate::BitsPerSec(kTargetBitrateBps),
3550 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003551
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003552 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003553 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003554 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003555 video_stream_encoder_->SetSource(&source,
3556 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003557
Åsa Persson8c1bf952018-09-13 10:42:19 +02003558 int64_t timestamp_ms = kFrameIntervalMs;
3559 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003560 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003561 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003562 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3563 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564
3565 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003566 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003567 timestamp_ms += kFrameIntervalMs;
3568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3569 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003570 EXPECT_THAT(source.sink_wants(),
3571 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3573 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3574
3575 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003576 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003577 timestamp_ms += kFrameIntervalMs;
3578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003579 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003580 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3582 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3583
3584 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003585 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003586 timestamp_ms += kFrameIntervalMs;
3587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3588 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003589 EXPECT_THAT(source.sink_wants(),
3590 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3592 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3593
3594 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003595 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003596 timestamp_ms += kFrameIntervalMs;
3597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003598 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003599 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003600 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3601 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3602
mflodmancc3d4422017-08-03 08:27:51 -07003603 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003604}
3605
Sergey Silkin41c650b2019-10-14 13:12:19 +02003606TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3607 fake_encoder_.SetResolutionBitrateLimits(
3608 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3609
Henrik Boström381d1092020-05-12 18:49:07 +02003610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003611 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3612 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3613 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3614 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003615
3616 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003617 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003618 source.set_adaptation_enabled(true);
3619 video_stream_encoder_->SetSource(
3620 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3621
3622 // Insert 720p frame.
3623 int64_t timestamp_ms = kFrameIntervalMs;
3624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3625 WaitForEncodedFrame(1280, 720);
3626
3627 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003629 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3630 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3631 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3632 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003633 video_stream_encoder_->TriggerQualityLow();
3634
3635 // Insert 720p frame. It should be downscaled and encoded.
3636 timestamp_ms += kFrameIntervalMs;
3637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3638 WaitForEncodedFrame(960, 540);
3639
3640 // Trigger adapt up. Higher resolution should not be requested duo to lack
3641 // of bitrate.
3642 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003643 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003644
3645 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003647 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3648 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3649 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3650 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003651
3652 // Trigger adapt up. Higher resolution should be requested.
3653 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003654 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003655
3656 video_stream_encoder_->Stop();
3657}
3658
3659TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3660 fake_encoder_.SetResolutionBitrateLimits(
3661 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3662
3663 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003665 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3666 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3667 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3668 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003669
3670 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003671 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003672 source.set_adaptation_enabled(true);
3673 video_stream_encoder_->SetSource(
3674 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3675
3676 // Insert 720p frame. It should be dropped and lower resolution should be
3677 // requested.
3678 int64_t timestamp_ms = kFrameIntervalMs;
3679 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3680 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003681 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003682
3683 // Insert 720p frame. It should be downscaled and encoded.
3684 timestamp_ms += kFrameIntervalMs;
3685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3686 WaitForEncodedFrame(960, 540);
3687
3688 video_stream_encoder_->Stop();
3689}
3690
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003691class BalancedDegradationTest : public VideoStreamEncoderTest {
3692 protected:
3693 void SetupTest() {
3694 // Reset encoder for field trials to take effect.
3695 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003696 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003697
3698 // Enable BALANCED preference.
3699 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003700 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3701 }
3702
3703 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003705 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3706 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003707 }
3708
Åsa Persson45b176f2019-09-30 11:19:05 +02003709 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003710 timestamp_ms_ += kFrameIntervalMs;
3711 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003712 }
3713
3714 void InsertFrameAndWaitForEncoded() {
3715 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003716 sink_.WaitForEncodedFrame(timestamp_ms_);
3717 }
3718
3719 const int kWidth = 640; // pixels:640x360=230400
3720 const int kHeight = 360;
3721 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3722 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003723 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003724};
3725
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003726TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003727 test::ScopedFieldTrials field_trials(
3728 "WebRTC-Video-BalancedDegradationSettings/"
3729 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3730 SetupTest();
3731
3732 // Force input frame rate.
3733 const int kInputFps = 24;
3734 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3735 stats.input_frame_rate = kInputFps;
3736 stats_proxy_->SetMockStats(stats);
3737
Åsa Persson45b176f2019-09-30 11:19:05 +02003738 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003739 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003740
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003741 // Trigger adapt down, expect scaled down framerate and resolution,
3742 // since Fps diff (input-requested:0) < threshold.
3743 video_stream_encoder_->TriggerQualityLow();
3744 EXPECT_THAT(source_.sink_wants(),
3745 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003746
3747 video_stream_encoder_->Stop();
3748}
3749
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003750TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003751 test::ScopedFieldTrials field_trials(
3752 "WebRTC-Video-BalancedDegradationSettings/"
3753 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3754 SetupTest();
3755
3756 // Force input frame rate.
3757 const int kInputFps = 25;
3758 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3759 stats.input_frame_rate = kInputFps;
3760 stats_proxy_->SetMockStats(stats);
3761
Åsa Persson45b176f2019-09-30 11:19:05 +02003762 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003763 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003764
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003765 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3766 // Fps diff (input-requested:1) == threshold.
3767 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003768 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003769
3770 video_stream_encoder_->Stop();
3771}
3772
3773TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3774 test::ScopedFieldTrials field_trials(
3775 "WebRTC-Video-BalancedDegradationSettings/"
3776 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3777 SetupTest();
3778
3779 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3780
Åsa Persson45b176f2019-09-30 11:19:05 +02003781 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003782 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003783
3784 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3785 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003786 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003787
3788 video_stream_encoder_->Stop();
3789}
3790
Åsa Perssonccfb3402019-09-25 15:13:04 +02003791TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003792 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003793 "WebRTC-Video-BalancedDegradationSettings/"
3794 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003795 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003796
Åsa Persson1b247f12019-08-14 17:26:39 +02003797 const int kMinBitrateBps = 425000;
3798 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003799 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003800
Åsa Persson45b176f2019-09-30 11:19:05 +02003801 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003802 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003803 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3804
3805 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3806 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003807 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003808 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003809 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3810
3811 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3812 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003813 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003814 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003815 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3816
Åsa Persson30ab0152019-08-27 12:22:33 +02003817 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3818 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003819 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003820 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003821 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003822 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3823
3824 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003825 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003826 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003827 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003828
Åsa Persson30ab0152019-08-27 12:22:33 +02003829 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003830 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003831 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003832 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003833 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003834 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3835
3836 video_stream_encoder_->Stop();
3837}
3838
Åsa Perssonccfb3402019-09-25 15:13:04 +02003839TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003840 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3841 test::ScopedFieldTrials field_trials(
3842 "WebRTC-Video-BalancedDegradationSettings/"
3843 "pixels:57600|129600|230400,fps:7|24|24/");
3844 SetupTest();
3845 OnBitrateUpdated(kLowTargetBitrateBps);
3846
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003847 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003848
3849 // Insert frame, expect scaled down:
3850 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3851 InsertFrame();
3852 EXPECT_FALSE(WaitForFrame(1000));
3853 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3854 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3855
3856 // Insert frame, expect scaled down:
3857 // resolution (320x180@24fps).
3858 InsertFrame();
3859 EXPECT_FALSE(WaitForFrame(1000));
3860 EXPECT_LT(source_.sink_wants().max_pixel_count,
3861 source_.last_wants().max_pixel_count);
3862 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3863
3864 // Frame should not be dropped (min pixels per frame reached).
3865 InsertFrameAndWaitForEncoded();
3866
3867 video_stream_encoder_->Stop();
3868}
3869
3870TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003871 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003872 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003873 "WebRTC-Video-BalancedDegradationSettings/"
3874 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003875 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003876
Åsa Persson30ab0152019-08-27 12:22:33 +02003877 const int kResolutionMinBitrateBps = 435000;
3878 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003879 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003880
Åsa Persson45b176f2019-09-30 11:19:05 +02003881 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003882 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003883 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3884
3885 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3886 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003887 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003888 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003889 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3890
3891 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3892 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003893 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003894 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003895 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3896
3897 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3898 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003899 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003900 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003901 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3902
Åsa Persson30ab0152019-08-27 12:22:33 +02003903 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3904 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003905 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003906 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003907 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3908
3909 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3910 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003911 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003912 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3913
3914 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003915 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003916 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003917 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003918 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003919 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3920
3921 video_stream_encoder_->Stop();
3922}
3923
Åsa Perssonccfb3402019-09-25 15:13:04 +02003924TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003925 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003926 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003927 "WebRTC-Video-BalancedDegradationSettings/"
3928 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003929 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003930
Åsa Persson30ab0152019-08-27 12:22:33 +02003931 const int kMinBitrateBps = 425000;
3932 const int kTooLowMinBitrateBps = 424000;
3933 const int kResolutionMinBitrateBps = 435000;
3934 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003935 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003936
Åsa Persson45b176f2019-09-30 11:19:05 +02003937 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003938 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003939 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3940
3941 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3942 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003943 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003944 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003945 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3946
3947 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3948 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003949 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003950 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003951 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3952
3953 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3954 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003955 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003956 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003957 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3958
3959 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3960 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003961 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003962 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3963
3964 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003965 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003966 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003967 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003968 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003969 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3970
3971 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003972 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003973 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003974 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003975 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3976
3977 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003978 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003979 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003980 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003981 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003982 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3983
Åsa Persson1b247f12019-08-14 17:26:39 +02003984 video_stream_encoder_->Stop();
3985}
3986
mflodmancc3d4422017-08-03 08:27:51 -07003987TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003988 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3989 const int kWidth = 1280;
3990 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003992 DataRate::BitsPerSec(kTargetBitrateBps),
3993 DataRate::BitsPerSec(kTargetBitrateBps),
3994 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003995
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003996 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003997 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003998 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003999 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004000 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004001
Åsa Persson8c1bf952018-09-13 10:42:19 +02004002 int64_t timestamp_ms = kFrameIntervalMs;
4003 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004004 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004005 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004006 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4007 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4008 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4009 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4010
4011 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004012 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004013 timestamp_ms += kFrameIntervalMs;
4014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4015 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004016 EXPECT_THAT(source.sink_wants(),
4017 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004018 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4019 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4020 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4021 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4022
4023 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004024 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004025 timestamp_ms += kFrameIntervalMs;
4026 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4027 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004028 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004029 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4030 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4031 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4032 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4033
Jonathan Yubc771b72017-12-08 17:04:29 -08004034 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004036 timestamp_ms += kFrameIntervalMs;
4037 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4038 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004039 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004040 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004042 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004043 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4044
Jonathan Yubc771b72017-12-08 17:04:29 -08004045 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004046 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004047 timestamp_ms += kFrameIntervalMs;
4048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4049 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004051 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004052 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4053 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4054 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4055 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4056
Jonathan Yubc771b72017-12-08 17:04:29 -08004057 // Trigger quality adapt down, expect no change (min resolution reached).
4058 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004059 timestamp_ms += kFrameIntervalMs;
4060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4061 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004062 EXPECT_THAT(source.sink_wants(), FpsMax());
4063 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004064 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4065 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4066 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4067 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4068
Evan Shrubsole64469032020-06-11 10:45:29 +02004069 // Trigger quality adapt up, expect upscaled resolution (480x270).
4070 video_stream_encoder_->TriggerQualityHigh();
4071 timestamp_ms += kFrameIntervalMs;
4072 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4073 WaitForEncodedFrame(timestamp_ms);
4074 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4077 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4078 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4079
4080 // Trigger quality and cpu adapt up since both are most limited, expect
4081 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004082 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004083 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004084 timestamp_ms += kFrameIntervalMs;
4085 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4086 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004087 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004088 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4089 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4090 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004091 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004092
Evan Shrubsole64469032020-06-11 10:45:29 +02004093 // Trigger quality and cpu adapt up since both are most limited, expect
4094 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004095 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004096 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004097 timestamp_ms += kFrameIntervalMs;
4098 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4099 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004100 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004101 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004102 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004103 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004104 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4105 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004106
Evan Shrubsole64469032020-06-11 10:45:29 +02004107 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4108 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004109 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004110 timestamp_ms += kFrameIntervalMs;
4111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4112 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004113 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004114 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4115 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004116 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004117 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004118
4119 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004120 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004121 timestamp_ms += kFrameIntervalMs;
4122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004123 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004124 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004125 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004126 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4127 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004128 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004129 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004130
mflodmancc3d4422017-08-03 08:27:51 -07004131 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004132}
4133
mflodmancc3d4422017-08-03 08:27:51 -07004134TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004135 const int kWidth = 640;
4136 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004137
Henrik Boström381d1092020-05-12 18:49:07 +02004138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004139 DataRate::BitsPerSec(kTargetBitrateBps),
4140 DataRate::BitsPerSec(kTargetBitrateBps),
4141 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004142
perkj803d97f2016-11-01 11:45:46 -07004143 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004144 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004145 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004146 }
4147
mflodmancc3d4422017-08-03 08:27:51 -07004148 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004149 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004150 video_source_.IncomingCapturedFrame(CreateFrame(
4151 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004152 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004153 }
4154
mflodmancc3d4422017-08-03 08:27:51 -07004155 video_stream_encoder_->Stop();
4156 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004157 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004158
Ying Wangef3998f2019-12-09 13:06:53 +01004159 EXPECT_METRIC_EQ(
4160 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4161 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004162 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4163}
4164
mflodmancc3d4422017-08-03 08:27:51 -07004165TEST_F(VideoStreamEncoderTest,
4166 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004168 DataRate::BitsPerSec(kTargetBitrateBps),
4169 DataRate::BitsPerSec(kTargetBitrateBps),
4170 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004171 const int kWidth = 640;
4172 const int kHeight = 360;
4173
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004174 video_stream_encoder_->SetSource(&video_source_,
4175 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004176
4177 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4178 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004179 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004180 }
4181
mflodmancc3d4422017-08-03 08:27:51 -07004182 video_stream_encoder_->Stop();
4183 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004184 stats_proxy_.reset();
4185
4186 EXPECT_EQ(0,
4187 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4188}
4189
Per Kjellanderdcef6412020-10-07 15:09:05 +02004190TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4191 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004192 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004193 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004194
4195 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004196 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004197 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004198 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4199 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004200
Henrik Boström381d1092020-05-12 18:49:07 +02004201 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004202 DataRate::BitsPerSec(kLowTargetBitrateBps),
4203 DataRate::BitsPerSec(kLowTargetBitrateBps),
4204 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004205
sprang57c2fff2017-01-16 06:24:02 -08004206 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004207 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4208 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004209 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4210 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4211
Erik Språngd7329ca2019-02-21 21:19:53 +01004212 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004213 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004214 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004215
Per Kjellanderdcef6412020-10-07 15:09:05 +02004216 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004217 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004218 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4219 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004220 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004221 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004222
Per Kjellanderdcef6412020-10-07 15:09:05 +02004223 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004224 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004225 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004226 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004227 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4228 WaitForEncodedFrame(CurrentTimeMs());
4229 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004230 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004231 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004232
mflodmancc3d4422017-08-03 08:27:51 -07004233 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004234}
4235
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004236TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004237 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004238 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004239 kVideoLayersAllocation);
4240
4241 const int kDefaultFps = 30;
4242
4243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4244 DataRate::BitsPerSec(kLowTargetBitrateBps),
4245 DataRate::BitsPerSec(kLowTargetBitrateBps),
4246 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4247
4248 video_source_.IncomingCapturedFrame(
4249 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4250 WaitForEncodedFrame(CurrentTimeMs());
4251 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4252 VideoLayersAllocation last_layer_allocation =
4253 sink_.GetLastVideoLayersAllocation();
4254 // kLowTargetBitrateBps is only enough for one spatial layer.
4255 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4256
4257 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004258 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004259 // Check that encoder has been updated too, not just allocation observer.
4260 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4261 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4262
Erik Språng9d69cbe2020-10-22 17:44:42 +02004263 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004264 int number_of_layers_allocation = 1;
4265 const int64_t start_time_ms = CurrentTimeMs();
4266 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4267 video_source_.IncomingCapturedFrame(
4268 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4269 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004270 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4271 number_of_layers_allocation = sink_.number_of_layers_allocations();
4272 VideoLayersAllocation new_allocation =
4273 sink_.GetLastVideoLayersAllocation();
4274 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4275 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4276 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4277 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4278 .target_bitrate_per_temporal_layer,
4279 last_layer_allocation.active_spatial_layers[0]
4280 .target_bitrate_per_temporal_layer);
4281 last_layer_allocation = new_allocation;
4282 }
4283 }
4284 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4285 video_stream_encoder_->Stop();
4286}
4287
4288TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004289 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004290 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4291 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4292 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004293 VideoEncoderConfig video_encoder_config;
4294 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4295 /* num_streams*/ 3, &video_encoder_config);
4296 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4297 video_encoder_config.content_type =
4298 VideoEncoderConfig::ContentType::kRealtimeVideo;
4299 video_encoder_config.encoder_specific_settings =
4300 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4301 VideoEncoder::GetDefaultVp8Settings());
4302 for (auto& layer : video_encoder_config.simulcast_layers) {
4303 layer.num_temporal_layers = 2;
4304 }
4305 // Simulcast layers are used for enabling/disabling streams.
4306 video_encoder_config.simulcast_layers[0].active = true;
4307 video_encoder_config.simulcast_layers[1].active = false;
4308 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004309 ConfigureEncoder(std::move(video_encoder_config),
4310 VideoStreamEncoder::BitrateAllocationCallbackType::
4311 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004312
4313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4314 DataRate::BitsPerSec(kTargetBitrateBps),
4315 DataRate::BitsPerSec(kTargetBitrateBps),
4316 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4317
4318 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4319 WaitForEncodedFrame(CurrentTimeMs());
4320 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4321 VideoLayersAllocation last_layer_allocation =
4322 sink_.GetLastVideoLayersAllocation();
4323
4324 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4325 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4326 .target_bitrate_per_temporal_layer,
4327 SizeIs(2));
4328 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4329 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4330 video_stream_encoder_->Stop();
4331}
4332
4333TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004334 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004335 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4336 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4337 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004338 VideoEncoderConfig video_encoder_config;
4339 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4340 /* num_streams*/ 3, &video_encoder_config);
4341 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4342 video_encoder_config.content_type =
4343 VideoEncoderConfig::ContentType::kRealtimeVideo;
4344 video_encoder_config.encoder_specific_settings =
4345 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4346 VideoEncoder::GetDefaultVp8Settings());
4347 for (auto& layer : video_encoder_config.simulcast_layers) {
4348 layer.num_temporal_layers = 2;
4349 }
4350 // Simulcast layers are used for enabling/disabling streams.
4351 video_encoder_config.simulcast_layers[0].active = true;
4352 video_encoder_config.simulcast_layers[1].active = false;
4353 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004354 ConfigureEncoder(std::move(video_encoder_config),
4355 VideoStreamEncoder::BitrateAllocationCallbackType::
4356 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004357
4358 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4359 DataRate::BitsPerSec(kTargetBitrateBps),
4360 DataRate::BitsPerSec(kTargetBitrateBps),
4361 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4362
4363 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4364 WaitForEncodedFrame(CurrentTimeMs());
4365 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4366 VideoLayersAllocation last_layer_allocation =
4367 sink_.GetLastVideoLayersAllocation();
4368
4369 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4370 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4371 .target_bitrate_per_temporal_layer,
4372 SizeIs(2));
4373 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4374
4375 video_stream_encoder_->Stop();
4376}
4377
4378TEST_F(VideoStreamEncoderTest,
4379 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4380 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4381 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004382 VideoEncoderConfig video_encoder_config;
4383 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4384 /* num_streams*/ 1, &video_encoder_config);
4385 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4386 video_encoder_config.content_type =
4387 VideoEncoderConfig::ContentType::kRealtimeVideo;
4388 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4389 vp9_settings.numberOfSpatialLayers = 2;
4390 vp9_settings.numberOfTemporalLayers = 2;
4391 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4392 vp9_settings.automaticResizeOn = false;
4393 video_encoder_config.encoder_specific_settings =
4394 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4395 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004396 ConfigureEncoder(std::move(video_encoder_config),
4397 VideoStreamEncoder::BitrateAllocationCallbackType::
4398 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004399
4400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4401 DataRate::BitsPerSec(kTargetBitrateBps),
4402 DataRate::BitsPerSec(kTargetBitrateBps),
4403 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4404
4405 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4406 WaitForEncodedFrame(CurrentTimeMs());
4407 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4408 VideoLayersAllocation last_layer_allocation =
4409 sink_.GetLastVideoLayersAllocation();
4410
4411 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4412 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4413 .target_bitrate_per_temporal_layer,
4414 SizeIs(2));
4415 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4416 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4417 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4418 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4419 .target_bitrate_per_temporal_layer,
4420 SizeIs(2));
4421 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4422 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4423 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4424
4425 // Since full SVC is used, expect the top layer to utilize the full target
4426 // rate.
4427 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4428 .target_bitrate_per_temporal_layer[1],
4429 DataRate::BitsPerSec(kTargetBitrateBps));
4430 video_stream_encoder_->Stop();
4431}
4432
4433TEST_F(VideoStreamEncoderTest,
4434 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4435 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4436 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004437 VideoEncoderConfig video_encoder_config;
4438 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4439 /* num_streams*/ 1, &video_encoder_config);
4440 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4441 video_encoder_config.content_type =
4442 VideoEncoderConfig::ContentType::kRealtimeVideo;
4443 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4444 vp9_settings.numberOfSpatialLayers = 2;
4445 vp9_settings.numberOfTemporalLayers = 2;
4446 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4447 vp9_settings.automaticResizeOn = false;
4448 video_encoder_config.encoder_specific_settings =
4449 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4450 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004451 ConfigureEncoder(std::move(video_encoder_config),
4452 VideoStreamEncoder::BitrateAllocationCallbackType::
4453 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004454
4455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4456 DataRate::BitsPerSec(kTargetBitrateBps),
4457 DataRate::BitsPerSec(kTargetBitrateBps),
4458 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4459
4460 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4461 WaitForEncodedFrame(CurrentTimeMs());
4462 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4463 VideoLayersAllocation last_layer_allocation =
4464 sink_.GetLastVideoLayersAllocation();
4465
4466 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4467 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4468 .target_bitrate_per_temporal_layer,
4469 SizeIs(1));
4470 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4471 .target_bitrate_per_temporal_layer,
4472 SizeIs(1));
4473 // Since full SVC is used, expect the top layer to utilize the full target
4474 // rate.
4475 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4476 .target_bitrate_per_temporal_layer[0],
4477 DataRate::BitsPerSec(kTargetBitrateBps));
4478 video_stream_encoder_->Stop();
4479}
4480
4481TEST_F(VideoStreamEncoderTest,
4482 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4483 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4484 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004485 VideoEncoderConfig video_encoder_config;
4486 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4487 /* num_streams*/ 1, &video_encoder_config);
4488 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4489 video_encoder_config.content_type =
4490 VideoEncoderConfig::ContentType::kRealtimeVideo;
4491 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4492 vp9_settings.numberOfSpatialLayers = 2;
4493 vp9_settings.numberOfTemporalLayers = 2;
4494 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4495 vp9_settings.automaticResizeOn = false;
4496 video_encoder_config.encoder_specific_settings =
4497 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4498 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004499 ConfigureEncoder(std::move(video_encoder_config),
4500 VideoStreamEncoder::BitrateAllocationCallbackType::
4501 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004502
4503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4504 DataRate::BitsPerSec(kTargetBitrateBps),
4505 DataRate::BitsPerSec(kTargetBitrateBps),
4506 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4507
4508 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4509 WaitForEncodedFrame(CurrentTimeMs());
4510 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4511 VideoLayersAllocation last_layer_allocation =
4512 sink_.GetLastVideoLayersAllocation();
4513
4514 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4515 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4516 .target_bitrate_per_temporal_layer,
4517 SizeIs(2));
4518 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4519 .target_bitrate_per_temporal_layer,
4520 SizeIs(2));
4521 // Since KSVC is, spatial layers are independend except on key frames.
4522 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4523 .target_bitrate_per_temporal_layer[1],
4524 DataRate::BitsPerSec(kTargetBitrateBps));
4525 video_stream_encoder_->Stop();
4526}
4527
4528TEST_F(VideoStreamEncoderTest,
4529 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4530 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4531 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4532 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004533 VideoEncoderConfig video_encoder_config;
4534 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4535 /* num_streams*/ 1, &video_encoder_config);
4536 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4537 video_encoder_config.content_type =
4538 VideoEncoderConfig::ContentType::kRealtimeVideo;
4539 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4540 vp9_settings.numberOfSpatialLayers = 3;
4541 vp9_settings.numberOfTemporalLayers = 2;
4542 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4543 vp9_settings.automaticResizeOn = false;
4544 video_encoder_config.encoder_specific_settings =
4545 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4546 vp9_settings);
4547 // Simulcast layers are used for enabling/disabling streams.
4548 video_encoder_config.simulcast_layers.resize(3);
4549 video_encoder_config.simulcast_layers[0].active = false;
4550 video_encoder_config.simulcast_layers[1].active = true;
4551 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004552 ConfigureEncoder(std::move(video_encoder_config),
4553 VideoStreamEncoder::BitrateAllocationCallbackType::
4554 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004555
4556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4557 DataRate::BitsPerSec(kTargetBitrateBps),
4558 DataRate::BitsPerSec(kTargetBitrateBps),
4559 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4560
4561 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4562 WaitForEncodedFrame(CurrentTimeMs());
4563 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4564 VideoLayersAllocation last_layer_allocation =
4565 sink_.GetLastVideoLayersAllocation();
4566
4567 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4568 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4569 .target_bitrate_per_temporal_layer,
4570 SizeIs(2));
4571 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4572 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4573
4574 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4575 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4576 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4577 .target_bitrate_per_temporal_layer,
4578 SizeIs(2));
4579 // Since full SVC is used, expect the top layer to utilize the full target
4580 // rate.
4581 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4582 .target_bitrate_per_temporal_layer[1],
4583 DataRate::BitsPerSec(kTargetBitrateBps));
4584 video_stream_encoder_->Stop();
4585}
4586
4587TEST_F(VideoStreamEncoderTest,
4588 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4589 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4590 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4591 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004592 VideoEncoderConfig video_encoder_config;
4593 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4594 /* num_streams*/ 1, &video_encoder_config);
4595 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4596 video_encoder_config.content_type =
4597 VideoEncoderConfig::ContentType::kRealtimeVideo;
4598 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4599 vp9_settings.numberOfSpatialLayers = 3;
4600 vp9_settings.numberOfTemporalLayers = 2;
4601 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4602 vp9_settings.automaticResizeOn = false;
4603 video_encoder_config.encoder_specific_settings =
4604 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4605 vp9_settings);
4606 // Simulcast layers are used for enabling/disabling streams.
4607 video_encoder_config.simulcast_layers.resize(3);
4608 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004609 ConfigureEncoder(std::move(video_encoder_config),
4610 VideoStreamEncoder::BitrateAllocationCallbackType::
4611 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004612
4613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4614 DataRate::BitsPerSec(kTargetBitrateBps),
4615 DataRate::BitsPerSec(kTargetBitrateBps),
4616 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4617
4618 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4619 WaitForEncodedFrame(CurrentTimeMs());
4620 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4621 VideoLayersAllocation last_layer_allocation =
4622 sink_.GetLastVideoLayersAllocation();
4623
4624 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4625 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4626 .target_bitrate_per_temporal_layer,
4627 SizeIs(2));
4628 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4629 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4630
4631 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4632 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4633 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4634 .target_bitrate_per_temporal_layer,
4635 SizeIs(2));
4636 video_stream_encoder_->Stop();
4637}
4638
4639TEST_F(VideoStreamEncoderTest,
4640 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4641 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4642 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4643 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004644 VideoEncoderConfig video_encoder_config;
4645 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4646 /* num_streams*/ 1, &video_encoder_config);
4647 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4648 video_encoder_config.content_type =
4649 VideoEncoderConfig::ContentType::kRealtimeVideo;
4650 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4651 vp9_settings.numberOfSpatialLayers = 3;
4652 vp9_settings.numberOfTemporalLayers = 2;
4653 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4654 vp9_settings.automaticResizeOn = false;
4655 video_encoder_config.encoder_specific_settings =
4656 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4657 vp9_settings);
4658 // Simulcast layers are used for enabling/disabling streams.
4659 video_encoder_config.simulcast_layers.resize(3);
4660 video_encoder_config.simulcast_layers[0].active = false;
4661 video_encoder_config.simulcast_layers[1].active = false;
4662 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004663 ConfigureEncoder(std::move(video_encoder_config),
4664 VideoStreamEncoder::BitrateAllocationCallbackType::
4665 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004666
4667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4668 DataRate::BitsPerSec(kTargetBitrateBps),
4669 DataRate::BitsPerSec(kTargetBitrateBps),
4670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4671
4672 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4673 WaitForEncodedFrame(CurrentTimeMs());
4674 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4675 VideoLayersAllocation last_layer_allocation =
4676 sink_.GetLastVideoLayersAllocation();
4677
4678 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4679 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4680 .target_bitrate_per_temporal_layer,
4681 SizeIs(2));
4682 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4683 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4684 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4685 .target_bitrate_per_temporal_layer[1],
4686 DataRate::BitsPerSec(kTargetBitrateBps));
4687 video_stream_encoder_->Stop();
4688}
4689
4690TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4691 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004692 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004693 kVideoLayersAllocation);
4694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4695 DataRate::BitsPerSec(kTargetBitrateBps),
4696 DataRate::BitsPerSec(kTargetBitrateBps),
4697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4698
4699 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4700 WaitForEncodedFrame(CurrentTimeMs());
4701 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4702 VideoLayersAllocation last_layer_allocation =
4703 sink_.GetLastVideoLayersAllocation();
4704
4705 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4706 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4707 .target_bitrate_per_temporal_layer,
4708 SizeIs(1));
4709 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4710 .target_bitrate_per_temporal_layer[0],
4711 DataRate::BitsPerSec(kTargetBitrateBps));
4712 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4713 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4714 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4715 video_stream_encoder_->Stop();
4716}
4717
4718TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004719 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4720 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004721 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004722 kVideoLayersAllocation);
4723
4724 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4725 DataRate::BitsPerSec(kLowTargetBitrateBps),
4726 DataRate::BitsPerSec(kLowTargetBitrateBps),
4727 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4728
4729 video_source_.IncomingCapturedFrame(
4730 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4731 WaitForEncodedFrame(CurrentTimeMs());
4732 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4733 VideoLayersAllocation last_layer_allocation =
4734 sink_.GetLastVideoLayersAllocation();
4735 // kLowTargetBitrateBps is only enough for one spatial layer.
4736 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4737 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4738 .target_bitrate_per_temporal_layer[0],
4739 DataRate::BitsPerSec(kLowTargetBitrateBps));
4740
4741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4742 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4743 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4744 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4745 video_source_.IncomingCapturedFrame(
4746 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4747 WaitForEncodedFrame(CurrentTimeMs());
4748
4749 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4750 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4751 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4752 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4753 .target_bitrate_per_temporal_layer[0],
4754 DataRate::Zero());
4755
4756 video_stream_encoder_->Stop();
4757}
4758
Per Kjellander4190ce92020-12-15 17:24:55 +01004759TEST_F(VideoStreamEncoderTest,
4760 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4761 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004762 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004763 kVideoLayersAllocation);
4764
4765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4766 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4767 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4768 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4769
4770 video_source_.IncomingCapturedFrame(
4771 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4772 WaitForEncodedFrame(CurrentTimeMs());
4773 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4774 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4775 SizeIs(2));
4776 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4777 codec_width_);
4778 EXPECT_EQ(
4779 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4780 codec_height_);
4781
4782 video_source_.IncomingCapturedFrame(
4783 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4784 WaitForEncodedFrame(CurrentTimeMs());
4785 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4786 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4787 SizeIs(2));
4788 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4789 codec_width_ / 2);
4790 EXPECT_EQ(
4791 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4792 codec_height_ / 2);
4793
4794 video_stream_encoder_->Stop();
4795}
4796
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004797TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4798 // 2 TLs configured, temporal layers supported by encoder.
4799 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004800 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004801 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004802 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004803 fake_encoder_.SetTemporalLayersSupported(0, true);
4804
4805 // Bitrate allocated across temporal layers.
4806 const int kTl0Bps = kTargetBitrateBps *
4807 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004808 kNumTemporalLayers, /*temporal_id*/ 0,
4809 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004810 const int kTl1Bps = kTargetBitrateBps *
4811 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004812 kNumTemporalLayers, /*temporal_id*/ 1,
4813 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004814 VideoBitrateAllocation expected_bitrate;
4815 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4816 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4817
4818 VerifyAllocatedBitrate(expected_bitrate);
4819 video_stream_encoder_->Stop();
4820}
4821
4822TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4823 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004824 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004825 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004826 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004827 fake_encoder_.SetTemporalLayersSupported(0, false);
4828
4829 // Temporal layers not supported by the encoder.
4830 // Total bitrate should be at ti:0.
4831 VideoBitrateAllocation expected_bitrate;
4832 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4833
4834 VerifyAllocatedBitrate(expected_bitrate);
4835 video_stream_encoder_->Stop();
4836}
4837
4838TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004839 webrtc::test::ScopedFieldTrials field_trials(
4840 "WebRTC-Video-QualityScalerSettings/"
4841 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4842 // Reset encoder for field trials to take effect.
4843 ConfigureEncoder(video_encoder_config_.Copy());
4844
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004845 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004846 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004847 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004848 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004849 fake_encoder_.SetTemporalLayersSupported(0, true);
4850 fake_encoder_.SetTemporalLayersSupported(1, false);
4851
4852 const int kS0Bps = 150000;
4853 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004854 kS0Bps *
4855 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4856 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004857 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004858 kS0Bps *
4859 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4860 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004861 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4862 // Temporal layers not supported by si:1.
4863 VideoBitrateAllocation expected_bitrate;
4864 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4865 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4866 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4867
4868 VerifyAllocatedBitrate(expected_bitrate);
4869 video_stream_encoder_->Stop();
4870}
4871
Niels Möller7dc26b72017-12-06 10:27:48 +01004872TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4873 const int kFrameWidth = 1280;
4874 const int kFrameHeight = 720;
4875 const int kFramerate = 24;
4876
Henrik Boström381d1092020-05-12 18:49:07 +02004877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004878 DataRate::BitsPerSec(kTargetBitrateBps),
4879 DataRate::BitsPerSec(kTargetBitrateBps),
4880 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004881 test::FrameForwarder source;
4882 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004883 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004884
4885 // Insert a single frame, triggering initial configuration.
4886 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4887 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4888
4889 EXPECT_EQ(
4890 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4891 kDefaultFramerate);
4892
4893 // Trigger reconfigure encoder (without resetting the entire instance).
4894 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004895 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4896 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004897 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004898 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004899 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004900 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4901
4902 // Detector should be updated with fps limit from codec config.
4903 EXPECT_EQ(
4904 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4905 kFramerate);
4906
4907 // Trigger overuse, max framerate should be reduced.
4908 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4909 stats.input_frame_rate = kFramerate;
4910 stats_proxy_->SetMockStats(stats);
4911 video_stream_encoder_->TriggerCpuOveruse();
4912 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4913 int adapted_framerate =
4914 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4915 EXPECT_LT(adapted_framerate, kFramerate);
4916
4917 // Trigger underuse, max framerate should go back to codec configured fps.
4918 // Set extra low fps, to make sure it's actually reset, not just incremented.
4919 stats = stats_proxy_->GetStats();
4920 stats.input_frame_rate = adapted_framerate / 2;
4921 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004922 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004923 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4924 EXPECT_EQ(
4925 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4926 kFramerate);
4927
4928 video_stream_encoder_->Stop();
4929}
4930
4931TEST_F(VideoStreamEncoderTest,
4932 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
4933 const int kFrameWidth = 1280;
4934 const int kFrameHeight = 720;
4935 const int kLowFramerate = 15;
4936 const int kHighFramerate = 25;
4937
Henrik Boström381d1092020-05-12 18:49:07 +02004938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004939 DataRate::BitsPerSec(kTargetBitrateBps),
4940 DataRate::BitsPerSec(kTargetBitrateBps),
4941 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004942 test::FrameForwarder source;
4943 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004944 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004945
4946 // Trigger initial configuration.
4947 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004948 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4949 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004950 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004951 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02004952 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02004953 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004954 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4955
4956 EXPECT_EQ(
4957 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4958 kLowFramerate);
4959
4960 // Trigger overuse, max framerate should be reduced.
4961 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4962 stats.input_frame_rate = kLowFramerate;
4963 stats_proxy_->SetMockStats(stats);
4964 video_stream_encoder_->TriggerCpuOveruse();
4965 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4966 int adapted_framerate =
4967 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4968 EXPECT_LT(adapted_framerate, kLowFramerate);
4969
4970 // Reconfigure the encoder with a new (higher max framerate), max fps should
4971 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02004972 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004973 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4974 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004975 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4977
4978 EXPECT_EQ(
4979 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4980 adapted_framerate);
4981
4982 // Trigger underuse, max framerate should go back to codec configured fps.
4983 stats = stats_proxy_->GetStats();
4984 stats.input_frame_rate = adapted_framerate;
4985 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004986 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004987 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4988 EXPECT_EQ(
4989 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4990 kHighFramerate);
4991
4992 video_stream_encoder_->Stop();
4993}
4994
mflodmancc3d4422017-08-03 08:27:51 -07004995TEST_F(VideoStreamEncoderTest,
4996 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07004997 const int kFrameWidth = 1280;
4998 const int kFrameHeight = 720;
4999 const int kFramerate = 24;
5000
Henrik Boström381d1092020-05-12 18:49:07 +02005001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005002 DataRate::BitsPerSec(kTargetBitrateBps),
5003 DataRate::BitsPerSec(kTargetBitrateBps),
5004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005005 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005007 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005008
5009 // Trigger initial configuration.
5010 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005011 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5012 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005013 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005014 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005015 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005016 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005018
Niels Möller7dc26b72017-12-06 10:27:48 +01005019 EXPECT_EQ(
5020 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5021 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005022
5023 // Trigger overuse, max framerate should be reduced.
5024 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5025 stats.input_frame_rate = kFramerate;
5026 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005027 video_stream_encoder_->TriggerCpuOveruse();
5028 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005029 int adapted_framerate =
5030 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005031 EXPECT_LT(adapted_framerate, kFramerate);
5032
5033 // Change degradation preference to not enable framerate scaling. Target
5034 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005035 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005036 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005037 EXPECT_EQ(
5038 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5039 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005040
mflodmancc3d4422017-08-03 08:27:51 -07005041 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005042}
5043
mflodmancc3d4422017-08-03 08:27:51 -07005044TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005045 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005047 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5048 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5049 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005050 const int kWidth = 640;
5051 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005052
asaperssonfab67072017-04-04 05:51:49 -07005053 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005054
5055 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005056 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005057
5058 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005059 EXPECT_TRUE_WAIT(
5060 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005061
sprangc5d62e22017-04-02 23:53:04 -07005062 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005063
asaperssonfab67072017-04-04 05:51:49 -07005064 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005065 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005066 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005067
5068 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005069 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005070
Henrik Boström2671dac2020-05-19 16:29:09 +02005071 EXPECT_TRUE_WAIT(
5072 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005073
mflodmancc3d4422017-08-03 08:27:51 -07005074 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005075}
5076
mflodmancc3d4422017-08-03 08:27:51 -07005077TEST_F(VideoStreamEncoderTest,
5078 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005079 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005081 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5082 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5083 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005084 const int kWidth = 640;
5085 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005086
5087 // We expect the n initial frames to get dropped.
5088 int i;
5089 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005090 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005091 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005092 }
5093 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005094 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005095 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005096
5097 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005098 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005099
mflodmancc3d4422017-08-03 08:27:51 -07005100 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005101}
5102
mflodmancc3d4422017-08-03 08:27:51 -07005103TEST_F(VideoStreamEncoderTest,
5104 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005105 const int kWidth = 640;
5106 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005108 DataRate::BitsPerSec(kLowTargetBitrateBps),
5109 DataRate::BitsPerSec(kLowTargetBitrateBps),
5110 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005111
5112 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005113 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005114 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005115
asaperssonfab67072017-04-04 05:51:49 -07005116 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005117 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005118 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005119
mflodmancc3d4422017-08-03 08:27:51 -07005120 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005121}
5122
mflodmancc3d4422017-08-03 08:27:51 -07005123TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005124 const int kWidth = 640;
5125 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005126 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005127
5128 VideoEncoderConfig video_encoder_config;
5129 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5130 // Make format different, to force recreation of encoder.
5131 video_encoder_config.video_format.parameters["foo"] = "foo";
5132 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005133 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005135 DataRate::BitsPerSec(kLowTargetBitrateBps),
5136 DataRate::BitsPerSec(kLowTargetBitrateBps),
5137 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005138
kthelgasonb83797b2017-02-14 11:57:25 -08005139 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005140 video_stream_encoder_->SetSource(&video_source_,
5141 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005142
asaperssonfab67072017-04-04 05:51:49 -07005143 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005144 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005145 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005146
mflodmancc3d4422017-08-03 08:27:51 -07005147 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005148 fake_encoder_.SetQualityScaling(true);
5149}
5150
Åsa Persson139f4dc2019-08-02 09:29:58 +02005151TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5152 webrtc::test::ScopedFieldTrials field_trials(
5153 "WebRTC-Video-QualityScalerSettings/"
5154 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5155 // Reset encoder for field trials to take effect.
5156 ConfigureEncoder(video_encoder_config_.Copy());
5157 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5158 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5159 const int kWidth = 640;
5160 const int kHeight = 360;
5161
Henrik Boström381d1092020-05-12 18:49:07 +02005162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005163 DataRate::BitsPerSec(kTargetBitrateBps),
5164 DataRate::BitsPerSec(kTargetBitrateBps),
5165 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005166 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5167 // Frame should not be dropped.
5168 WaitForEncodedFrame(1);
5169
Henrik Boström381d1092020-05-12 18:49:07 +02005170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005171 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5172 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5173 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005174 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5175 // Frame should not be dropped.
5176 WaitForEncodedFrame(2);
5177
Henrik Boström381d1092020-05-12 18:49:07 +02005178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005179 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5180 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5181 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005182 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5183 // Expect to drop this frame, the wait should time out.
5184 ExpectDroppedFrame();
5185
5186 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005187 EXPECT_TRUE_WAIT(
5188 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005189 video_stream_encoder_->Stop();
5190}
5191
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005192TEST_F(VideoStreamEncoderTest,
5193 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5194 webrtc::test::ScopedFieldTrials field_trials(
5195 "WebRTC-Video-QualityScalerSettings/"
5196 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5197 fake_encoder_.SetQualityScaling(false);
5198 ConfigureEncoder(video_encoder_config_.Copy());
5199 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5200 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5201 const int kWidth = 640;
5202 const int kHeight = 360;
5203
5204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5205 DataRate::BitsPerSec(kTargetBitrateBps),
5206 DataRate::BitsPerSec(kTargetBitrateBps),
5207 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5208 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5209 // Frame should not be dropped.
5210 WaitForEncodedFrame(1);
5211
5212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5213 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5214 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5215 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5216 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5217 // Frame should not be dropped.
5218 WaitForEncodedFrame(2);
5219
5220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5221 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5222 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5223 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5224 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5225 // Not dropped since quality scaling is disabled.
5226 WaitForEncodedFrame(3);
5227
5228 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005229 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005230 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5231
5232 video_stream_encoder_->Stop();
5233}
5234
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005235TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5236 const int kLowTargetBitrateBps = 400000;
5237 // Set simulcast.
5238 ResetEncoder("VP8", 3, 1, 1, false);
5239 fake_encoder_.SetQualityScaling(true);
5240 const int kWidth = 1280;
5241 const int kHeight = 720;
5242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5243 DataRate::BitsPerSec(kLowTargetBitrateBps),
5244 DataRate::BitsPerSec(kLowTargetBitrateBps),
5245 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5246 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5247 // Frame should not be dropped.
5248 WaitForEncodedFrame(1);
5249
5250 // Trigger QVGA "singlecast"
5251 // Update the config.
5252 VideoEncoderConfig video_encoder_config;
5253 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5254 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005255 video_encoder_config.video_stream_factory =
5256 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5257 "VP8", /*max qp*/ 56, /*screencast*/ false,
5258 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005259 for (auto& layer : video_encoder_config.simulcast_layers) {
5260 layer.num_temporal_layers = 1;
5261 layer.max_framerate = kDefaultFramerate;
5262 }
5263 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5264 video_encoder_config.content_type =
5265 VideoEncoderConfig::ContentType::kRealtimeVideo;
5266
5267 video_encoder_config.simulcast_layers[0].active = true;
5268 video_encoder_config.simulcast_layers[1].active = false;
5269 video_encoder_config.simulcast_layers[2].active = false;
5270
5271 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5272 kMaxPayloadLength);
5273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5274
5275 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5276 // Frame should not be dropped.
5277 WaitForEncodedFrame(2);
5278
5279 // Trigger HD "singlecast"
5280 video_encoder_config.simulcast_layers[0].active = false;
5281 video_encoder_config.simulcast_layers[1].active = false;
5282 video_encoder_config.simulcast_layers[2].active = true;
5283
5284 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5285 kMaxPayloadLength);
5286 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5287
5288 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5289 // Frame should be dropped because of initial frame drop.
5290 ExpectDroppedFrame();
5291
5292 // Expect the sink_wants to specify a scaled frame.
5293 EXPECT_TRUE_WAIT(
5294 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5295 video_stream_encoder_->Stop();
5296}
5297
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005298TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5299 const int kLowTargetBitrateBps = 400000;
5300 // Set simulcast.
5301 ResetEncoder("VP9", 1, 1, 3, false);
5302 fake_encoder_.SetQualityScaling(true);
5303 const int kWidth = 1280;
5304 const int kHeight = 720;
5305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5306 DataRate::BitsPerSec(kLowTargetBitrateBps),
5307 DataRate::BitsPerSec(kLowTargetBitrateBps),
5308 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5309 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5310 // Frame should not be dropped.
5311 WaitForEncodedFrame(1);
5312
5313 // Trigger QVGA "singlecast"
5314 // Update the config.
5315 VideoEncoderConfig video_encoder_config;
5316 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5317 &video_encoder_config);
5318 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5319 vp9_settings.numberOfSpatialLayers = 3;
5320 // Since only one layer is active - automatic resize should be enabled.
5321 vp9_settings.automaticResizeOn = true;
5322 video_encoder_config.encoder_specific_settings =
5323 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5324 vp9_settings);
5325 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5326 video_encoder_config.content_type =
5327 VideoEncoderConfig::ContentType::kRealtimeVideo;
5328 // Currently simulcast layers |active| flags are used to inidicate
5329 // which SVC layers are active.
5330 video_encoder_config.simulcast_layers.resize(3);
5331
5332 video_encoder_config.simulcast_layers[0].active = true;
5333 video_encoder_config.simulcast_layers[1].active = false;
5334 video_encoder_config.simulcast_layers[2].active = false;
5335
5336 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5337 kMaxPayloadLength);
5338 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5339
5340 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5341 // Frame should not be dropped.
5342 WaitForEncodedFrame(2);
5343
5344 // Trigger HD "singlecast"
5345 video_encoder_config.simulcast_layers[0].active = false;
5346 video_encoder_config.simulcast_layers[1].active = false;
5347 video_encoder_config.simulcast_layers[2].active = true;
5348
5349 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5350 kMaxPayloadLength);
5351 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5352
5353 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5354 // Frame should be dropped because of initial frame drop.
5355 ExpectDroppedFrame();
5356
5357 // Expect the sink_wants to specify a scaled frame.
5358 EXPECT_TRUE_WAIT(
5359 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5360 video_stream_encoder_->Stop();
5361}
5362
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005363TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005364 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5365 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5366 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5367 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5368 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5369 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5370 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5371 fake_encoder_.SetResolutionBitrateLimits(
5372 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5373
5374 VideoEncoderConfig video_encoder_config;
5375 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5376 &video_encoder_config);
5377 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5378 vp9_settings.numberOfSpatialLayers = 3;
5379 // Since only one layer is active - automatic resize should be enabled.
5380 vp9_settings.automaticResizeOn = true;
5381 video_encoder_config.encoder_specific_settings =
5382 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5383 vp9_settings);
5384 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5385 video_encoder_config.content_type =
5386 VideoEncoderConfig::ContentType::kRealtimeVideo;
5387 // Simulcast layers are used to indicate which spatial layers are active.
5388 video_encoder_config.simulcast_layers.resize(3);
5389 video_encoder_config.simulcast_layers[0].active = false;
5390 video_encoder_config.simulcast_layers[1].active = true;
5391 video_encoder_config.simulcast_layers[2].active = false;
5392
5393 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5394 kMaxPayloadLength);
5395 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5396
5397 // The encoder bitrate limits for 360p should be used.
5398 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5399 EXPECT_FALSE(WaitForFrame(1000));
5400 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5401 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5402 VideoCodecType::kVideoCodecVP9);
5403 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5404 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5405 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5406 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5407 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5408 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5409 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5410 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5411
5412 // The encoder bitrate limits for 270p should be used.
5413 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5414 EXPECT_FALSE(WaitForFrame(1000));
5415 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5416 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5417 VideoCodecType::kVideoCodecVP9);
5418 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5419 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5420 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5421 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5422 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5423 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5424 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5425 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5426
5427 video_stream_encoder_->Stop();
5428}
5429
5430TEST_F(VideoStreamEncoderTest,
5431 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5432 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5433 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5434 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5435 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5436 fake_encoder_.SetResolutionBitrateLimits(
5437 {kEncoderLimits180p, kEncoderLimits720p});
5438
5439 VideoEncoderConfig video_encoder_config;
5440 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5441 &video_encoder_config);
5442 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5443 vp9_settings.numberOfSpatialLayers = 3;
5444 // Since only one layer is active - automatic resize should be enabled.
5445 vp9_settings.automaticResizeOn = true;
5446 video_encoder_config.encoder_specific_settings =
5447 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5448 vp9_settings);
5449 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5450 video_encoder_config.content_type =
5451 VideoEncoderConfig::ContentType::kRealtimeVideo;
5452 // Simulcast layers are used to indicate which spatial layers are active.
5453 video_encoder_config.simulcast_layers.resize(3);
5454 video_encoder_config.simulcast_layers[0].active = true;
5455 video_encoder_config.simulcast_layers[1].active = false;
5456 video_encoder_config.simulcast_layers[2].active = false;
5457
5458 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5459 kMaxPayloadLength);
5460 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5461
5462 // Limits not applied on lowest stream, limits for 180p should not be used.
5463 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5464 EXPECT_FALSE(WaitForFrame(1000));
5465 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5466 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5467 VideoCodecType::kVideoCodecVP9);
5468 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5469 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5470 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5471 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5472 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5473 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5474 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5475 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5476
5477 video_stream_encoder_->Stop();
5478}
5479
5480TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005481 InitialFrameDropActivatesWhenResolutionIncreases) {
5482 const int kWidth = 640;
5483 const int kHeight = 360;
5484
5485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5486 DataRate::BitsPerSec(kTargetBitrateBps),
5487 DataRate::BitsPerSec(kTargetBitrateBps),
5488 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5489 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5490 // Frame should not be dropped.
5491 WaitForEncodedFrame(1);
5492
5493 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5494 DataRate::BitsPerSec(kLowTargetBitrateBps),
5495 DataRate::BitsPerSec(kLowTargetBitrateBps),
5496 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5497 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5498 // Frame should not be dropped, bitrate not too low for frame.
5499 WaitForEncodedFrame(2);
5500
5501 // Incoming resolution increases.
5502 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5503 // Expect to drop this frame, bitrate too low for frame.
5504 ExpectDroppedFrame();
5505
5506 // Expect the sink_wants to specify a scaled frame.
5507 EXPECT_TRUE_WAIT(
5508 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5509 video_stream_encoder_->Stop();
5510}
5511
5512TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5513 const int kWidth = 640;
5514 const int kHeight = 360;
5515 // So that quality scaling doesn't happen by itself.
5516 fake_encoder_.SetQp(kQpHigh);
5517
5518 AdaptingFrameForwarder source(&time_controller_);
5519 source.set_adaptation_enabled(true);
5520 video_stream_encoder_->SetSource(
5521 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5522
5523 int timestamp = 1;
5524
5525 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5526 DataRate::BitsPerSec(kTargetBitrateBps),
5527 DataRate::BitsPerSec(kTargetBitrateBps),
5528 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5529 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5530 WaitForEncodedFrame(timestamp);
5531 timestamp += 9000;
5532 // Long pause to disable all first BWE drop logic.
5533 AdvanceTime(TimeDelta::Millis(1000));
5534
5535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5536 DataRate::BitsPerSec(kLowTargetBitrateBps),
5537 DataRate::BitsPerSec(kLowTargetBitrateBps),
5538 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5539 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5540 // Not dropped frame, as initial frame drop is disabled by now.
5541 WaitForEncodedFrame(timestamp);
5542 timestamp += 9000;
5543 AdvanceTime(TimeDelta::Millis(100));
5544
5545 // Quality adaptation down.
5546 video_stream_encoder_->TriggerQualityLow();
5547
5548 // Adaptation has an effect.
5549 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5550 5000);
5551
5552 // Frame isn't dropped as initial frame dropper is disabled.
5553 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5554 WaitForEncodedFrame(timestamp);
5555 timestamp += 9000;
5556 AdvanceTime(TimeDelta::Millis(100));
5557
5558 // Quality adaptation up.
5559 video_stream_encoder_->TriggerQualityHigh();
5560
5561 // Adaptation has an effect.
5562 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5563 5000);
5564
5565 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5566 // Frame should not be dropped, as initial framedropper is off.
5567 WaitForEncodedFrame(timestamp);
5568
5569 video_stream_encoder_->Stop();
5570}
5571
Åsa Persson7f354f82021-02-04 15:52:15 +01005572TEST_F(VideoStreamEncoderTest,
5573 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5574 const int kMinStartBps360p = 222000;
5575 fake_encoder_.SetResolutionBitrateLimits(
5576 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5577 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5578 800000)});
5579
5580 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5581 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5582 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5583 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5584 0, 0, 0);
5585 // Frame should not be dropped, bitrate not too low for frame.
5586 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5587 WaitForEncodedFrame(1);
5588
5589 // Incoming resolution increases, initial frame drop activates.
5590 // Frame should be dropped, link allocation too low for frame.
5591 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5592 ExpectDroppedFrame();
5593
5594 // Expect sink_wants to specify a scaled frame.
5595 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5596 5000);
5597 video_stream_encoder_->Stop();
5598}
5599
5600TEST_F(VideoStreamEncoderTest,
5601 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5602 const int kMinStartBps360p = 222000;
5603 fake_encoder_.SetResolutionBitrateLimits(
5604 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5605 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5606 800000)});
5607
5608 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5609 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5610 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5611 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5612 0, 0, 0);
5613 // Frame should not be dropped, bitrate not too low for frame.
5614 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5615 WaitForEncodedFrame(1);
5616
5617 // Incoming resolution increases, initial frame drop activates.
5618 // Frame should be dropped, link allocation not too low for frame.
5619 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5620 WaitForEncodedFrame(2);
5621
5622 video_stream_encoder_->Stop();
5623}
5624
Åsa Perssone644a032019-11-08 15:56:00 +01005625TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5626 webrtc::test::ScopedFieldTrials field_trials(
5627 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5628
5629 // Reset encoder for field trials to take effect.
5630 VideoEncoderConfig config = video_encoder_config_.Copy();
5631 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005632 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005633 ConfigureEncoder(std::move(config));
5634 fake_encoder_.SetQp(kQpLow);
5635
5636 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005637 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005638 source.set_adaptation_enabled(true);
5639 video_stream_encoder_->SetSource(&source,
5640 DegradationPreference::MAINTAIN_FRAMERATE);
5641
5642 // Start at low bitrate.
5643 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5645 DataRate::BitsPerSec(kLowBitrateBps),
5646 DataRate::BitsPerSec(kLowBitrateBps),
5647 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005648
5649 // Expect first frame to be dropped and resolution to be limited.
5650 const int kWidth = 1280;
5651 const int kHeight = 720;
5652 const int64_t kFrameIntervalMs = 100;
5653 int64_t timestamp_ms = kFrameIntervalMs;
5654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5655 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005656 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5657 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005658
5659 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5661 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005662
5663 // Insert frames and advance |min_duration_ms|.
5664 for (size_t i = 1; i <= 10; i++) {
5665 timestamp_ms += kFrameIntervalMs;
5666 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5667 WaitForEncodedFrame(timestamp_ms);
5668 }
5669 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5670 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5671
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005672 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005673
5674 // Insert frame should trigger high BW and release quality limitation.
5675 timestamp_ms += kFrameIntervalMs;
5676 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5677 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005678 // The ramp-up code involves the adaptation queue, give it time to execute.
5679 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005680 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005681 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005682
5683 // Frame should not be adapted.
5684 timestamp_ms += kFrameIntervalMs;
5685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5686 WaitForEncodedFrame(kWidth, kHeight);
5687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5688
5689 video_stream_encoder_->Stop();
5690}
5691
mflodmancc3d4422017-08-03 08:27:51 -07005692TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005693 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005694 webrtc::test::ScopedFieldTrials field_trials(
5695 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005696 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005697 source.set_adaptation_enabled(true);
5698 video_stream_encoder_->SetSource(&source,
5699 DegradationPreference::MAINTAIN_FRAMERATE);
5700 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5701 DataRate::BitsPerSec(kTargetBitrateBps),
5702 DataRate::BitsPerSec(kTargetBitrateBps),
5703 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5704 fake_encoder_.SetQp(kQpHigh + 1);
5705 const int kWidth = 1280;
5706 const int kHeight = 720;
5707 const int64_t kFrameIntervalMs = 100;
5708 int64_t timestamp_ms = kFrameIntervalMs;
5709 for (size_t i = 1; i <= 100; i++) {
5710 timestamp_ms += kFrameIntervalMs;
5711 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5712 WaitForEncodedFrame(timestamp_ms);
5713 }
5714 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5715 // for the first time.
5716 // TODO(eshr): We should avoid these waits by using threads with simulated
5717 // time.
5718 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5719 2000 * 2.5 * 2);
5720 timestamp_ms += kFrameIntervalMs;
5721 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5722 WaitForEncodedFrame(timestamp_ms);
5723 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5724 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5725 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5726
5727 // Disable Quality scaling by turning off scaler on the encoder and
5728 // reconfiguring.
5729 fake_encoder_.SetQualityScaling(false);
5730 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5731 kMaxPayloadLength);
5732 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005733 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005734 // Since we turned off the quality scaler, the adaptations made by it are
5735 // removed.
5736 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5738
5739 video_stream_encoder_->Stop();
5740}
5741
5742TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005743 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5744 const int kTooSmallWidth = 10;
5745 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005746 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005747 DataRate::BitsPerSec(kTargetBitrateBps),
5748 DataRate::BitsPerSec(kTargetBitrateBps),
5749 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005750
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005751 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005752 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005753 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005754 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005755 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005756 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5757
5758 // Trigger adapt down, too small frame, expect no change.
5759 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005760 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005761 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005762 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005763 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5764 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5765
mflodmancc3d4422017-08-03 08:27:51 -07005766 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005767}
5768
mflodmancc3d4422017-08-03 08:27:51 -07005769TEST_F(VideoStreamEncoderTest,
5770 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005771 const int kTooSmallWidth = 10;
5772 const int kTooSmallHeight = 10;
5773 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005775 DataRate::BitsPerSec(kTargetBitrateBps),
5776 DataRate::BitsPerSec(kTargetBitrateBps),
5777 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005778
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005779 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07005780 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005781 video_stream_encoder_->SetSource(&source,
5782 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005783 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07005784 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5785 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5786
5787 // Trigger adapt down, expect limited framerate.
5788 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005789 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005790 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005791 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005792 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5793 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5794 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5795
5796 // Trigger adapt down, too small frame, expect no change.
5797 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005798 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07005799 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005800 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005801 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5802 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5803 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5804
mflodmancc3d4422017-08-03 08:27:51 -07005805 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005806}
5807
mflodmancc3d4422017-08-03 08:27:51 -07005808TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07005809 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02005810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005811 DataRate::BitsPerSec(kTargetBitrateBps),
5812 DataRate::BitsPerSec(kTargetBitrateBps),
5813 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02005814 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07005815 const int kFrameWidth = 1280;
5816 const int kFrameHeight = 720;
5817 video_source_.IncomingCapturedFrame(
5818 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005819 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07005820 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07005821}
5822
sprangb1ca0732017-02-01 08:38:12 -08005823// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07005824TEST_F(VideoStreamEncoderTest,
5825 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02005826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005827 DataRate::BitsPerSec(kTargetBitrateBps),
5828 DataRate::BitsPerSec(kTargetBitrateBps),
5829 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08005830
5831 const int kFrameWidth = 1280;
5832 const int kFrameHeight = 720;
5833 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07005834 // requested by
5835 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08005836 video_source_.set_adaptation_enabled(true);
5837
5838 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005839 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005840 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005841
5842 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005843 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08005844 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005845 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005846 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08005847
asaperssonfab67072017-04-04 05:51:49 -07005848 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02005849 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08005850 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005851 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005852 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005853
mflodmancc3d4422017-08-03 08:27:51 -07005854 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08005855}
sprangfe627f32017-03-29 08:24:59 -07005856
mflodmancc3d4422017-08-03 08:27:51 -07005857TEST_F(VideoStreamEncoderTest,
5858 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07005859 const int kFrameWidth = 1280;
5860 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07005861
Henrik Boström381d1092020-05-12 18:49:07 +02005862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005863 DataRate::BitsPerSec(kTargetBitrateBps),
5864 DataRate::BitsPerSec(kTargetBitrateBps),
5865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005866 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005867 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005868 video_source_.set_adaptation_enabled(true);
5869
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005870 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005871
5872 video_source_.IncomingCapturedFrame(
5873 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005874 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005875
5876 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07005877 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005878
5879 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07005880 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005881 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005882 video_source_.IncomingCapturedFrame(
5883 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005884 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005885 }
5886
5887 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07005888 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005889 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005890 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005891 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005892 video_source_.IncomingCapturedFrame(
5893 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005894 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005895 ++num_frames_dropped;
5896 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005897 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005898 }
5899 }
5900
sprang4847ae62017-06-27 07:06:52 -07005901 // Add some slack to account for frames dropped by the frame dropper.
5902 const int kErrorMargin = 1;
5903 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005904 kErrorMargin);
5905
5906 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07005907 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005908 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02005909 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005910 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005911 video_source_.IncomingCapturedFrame(
5912 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005913 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005914 ++num_frames_dropped;
5915 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005916 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005917 }
5918 }
sprang4847ae62017-06-27 07:06:52 -07005919 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07005920 kErrorMargin);
5921
5922 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02005923 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005924 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005925 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005926 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005927 video_source_.IncomingCapturedFrame(
5928 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005929 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005930 ++num_frames_dropped;
5931 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005932 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005933 }
5934 }
sprang4847ae62017-06-27 07:06:52 -07005935 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005936 kErrorMargin);
5937
5938 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02005939 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005940 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005941 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005942 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005943 video_source_.IncomingCapturedFrame(
5944 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005945 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005946 ++num_frames_dropped;
5947 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005948 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005949 }
5950 }
5951 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
5952
mflodmancc3d4422017-08-03 08:27:51 -07005953 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005954}
5955
mflodmancc3d4422017-08-03 08:27:51 -07005956TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07005957 const int kFramerateFps = 5;
5958 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07005959 const int kFrameWidth = 1280;
5960 const int kFrameHeight = 720;
5961
sprang4847ae62017-06-27 07:06:52 -07005962 // Reconfigure encoder with two temporal layers and screensharing, which will
5963 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02005964 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07005965
Henrik Boström381d1092020-05-12 18:49:07 +02005966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005967 DataRate::BitsPerSec(kTargetBitrateBps),
5968 DataRate::BitsPerSec(kTargetBitrateBps),
5969 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005970 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005971 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005972 video_source_.set_adaptation_enabled(true);
5973
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005974 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005975
5976 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08005977 rtc::VideoSinkWants last_wants;
5978 do {
5979 last_wants = video_source_.sink_wants();
5980
sprangc5d62e22017-04-02 23:53:04 -07005981 // Insert frames to get a new fps estimate...
5982 for (int j = 0; j < kFramerateFps; ++j) {
5983 video_source_.IncomingCapturedFrame(
5984 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08005985 if (video_source_.last_sent_width()) {
5986 sink_.WaitForEncodedFrame(timestamp_ms);
5987 }
sprangc5d62e22017-04-02 23:53:04 -07005988 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005989 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07005990 }
5991 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07005992 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08005993 } while (video_source_.sink_wants().max_framerate_fps <
5994 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07005995
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005996 EXPECT_THAT(video_source_.sink_wants(),
5997 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07005998
mflodmancc3d4422017-08-03 08:27:51 -07005999 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006000}
asaperssonf7e294d2017-06-13 23:25:22 -07006001
mflodmancc3d4422017-08-03 08:27:51 -07006002TEST_F(VideoStreamEncoderTest,
6003 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006004 const int kWidth = 1280;
6005 const int kHeight = 720;
6006 const int64_t kFrameIntervalMs = 150;
6007 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006009 DataRate::BitsPerSec(kTargetBitrateBps),
6010 DataRate::BitsPerSec(kTargetBitrateBps),
6011 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006012
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006013 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006014 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006015 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006016 video_stream_encoder_->SetSource(&source,
6017 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006018 timestamp_ms += kFrameIntervalMs;
6019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006020 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006021 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6023 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6024 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6025
6026 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006027 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006028 timestamp_ms += kFrameIntervalMs;
6029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006030 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006031 EXPECT_THAT(source.sink_wants(),
6032 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006033 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6035 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6036
6037 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006038 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006039 timestamp_ms += kFrameIntervalMs;
6040 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006041 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006042 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006043 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6044 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6045 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6046
6047 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006048 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006049 timestamp_ms += kFrameIntervalMs;
6050 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006051 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006052 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006053 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6054 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6055 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6056
6057 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006058 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006059 timestamp_ms += kFrameIntervalMs;
6060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006061 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006062 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006063 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6064 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6065 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6066
6067 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006068 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006069 timestamp_ms += kFrameIntervalMs;
6070 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006071 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006072 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006073 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6074 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6075 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6076
6077 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006078 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006079 timestamp_ms += kFrameIntervalMs;
6080 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006081 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006082 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006083 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6084 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6085 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6086
6087 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006088 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006089 timestamp_ms += kFrameIntervalMs;
6090 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006091 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006092 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006093 rtc::VideoSinkWants last_wants = source.sink_wants();
6094 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6095 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6096 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6097
6098 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006099 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006100 timestamp_ms += kFrameIntervalMs;
6101 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006102 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006103 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006104 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6105 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6106 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6107
Evan Shrubsole64469032020-06-11 10:45:29 +02006108 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006109 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006110 timestamp_ms += kFrameIntervalMs;
6111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006112 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006113 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006114 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6115 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6116 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6117
6118 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006119 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006120 timestamp_ms += kFrameIntervalMs;
6121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006122 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006123 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006124 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6125 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6126 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6127
6128 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006129 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006130 timestamp_ms += kFrameIntervalMs;
6131 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006132 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006133 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006134 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6135 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6136 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6137
6138 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006139 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006140 timestamp_ms += kFrameIntervalMs;
6141 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006142 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006143 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006144 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6145 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6146 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6147
6148 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006149 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006150 timestamp_ms += kFrameIntervalMs;
6151 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006152 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006153 EXPECT_THAT(source.sink_wants(), FpsMax());
6154 EXPECT_EQ(source.sink_wants().max_pixel_count,
6155 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006156 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6157 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6158 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6159
6160 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006161 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006162 timestamp_ms += kFrameIntervalMs;
6163 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006164 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006165 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006166 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6167 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6168 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6169
Åsa Persson30ab0152019-08-27 12:22:33 +02006170 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006171 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006172 timestamp_ms += kFrameIntervalMs;
6173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006174 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006175 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006176 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006177 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6178 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6179 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6180
6181 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006182 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006183 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006184 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6185
mflodmancc3d4422017-08-03 08:27:51 -07006186 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006187}
6188
mflodmancc3d4422017-08-03 08:27:51 -07006189TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006190 const int kWidth = 1280;
6191 const int kHeight = 720;
6192 const int64_t kFrameIntervalMs = 150;
6193 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006194 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006195 DataRate::BitsPerSec(kTargetBitrateBps),
6196 DataRate::BitsPerSec(kTargetBitrateBps),
6197 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006198
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006199 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006200 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006201 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006202 video_stream_encoder_->SetSource(&source,
6203 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006204 timestamp_ms += kFrameIntervalMs;
6205 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006206 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006207 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6209 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6210 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6211 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6212 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6213 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6214
6215 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006216 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006217 timestamp_ms += kFrameIntervalMs;
6218 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006219 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006220 EXPECT_THAT(source.sink_wants(),
6221 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006222 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6223 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6224 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6225 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6226 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6227 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6228
6229 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006230 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006231 timestamp_ms += kFrameIntervalMs;
6232 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006233 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006234 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006235 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6237 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6238 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6239 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6240 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6241
6242 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006243 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006244 timestamp_ms += kFrameIntervalMs;
6245 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006246 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006247 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006248 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006249 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6250 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6251 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6252 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6253 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6254
Evan Shrubsole64469032020-06-11 10:45:29 +02006255 // Trigger cpu adapt up, expect no change since QP is most limited.
6256 {
6257 // Store current sink wants since we expect no change and if there is no
6258 // change then last_wants() is not updated.
6259 auto previous_sink_wants = source.sink_wants();
6260 video_stream_encoder_->TriggerCpuUnderuse();
6261 timestamp_ms += kFrameIntervalMs;
6262 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6263 WaitForEncodedFrame(timestamp_ms);
6264 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6265 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6266 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6267 }
6268
6269 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6270 video_stream_encoder_->TriggerQualityHigh();
6271 timestamp_ms += kFrameIntervalMs;
6272 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6273 WaitForEncodedFrame(timestamp_ms);
6274 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6275 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6276 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6277 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6278 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6279 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6280 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6281
6282 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6283 // expect increased resolution (960x540@30fps).
6284 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006285 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006286 timestamp_ms += kFrameIntervalMs;
6287 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006288 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006289 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006290 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6291 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6292 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6293 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6294 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006295 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006296
Evan Shrubsole64469032020-06-11 10:45:29 +02006297 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6298 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006299 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006300 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006301 timestamp_ms += kFrameIntervalMs;
6302 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006303 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006304 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006305 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006306 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6308 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6309 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6310 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006311 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006312
6313 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006314 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006315 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006316 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006317 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006318
mflodmancc3d4422017-08-03 08:27:51 -07006319 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006320}
6321
mflodmancc3d4422017-08-03 08:27:51 -07006322TEST_F(VideoStreamEncoderTest,
6323 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006324 const int kWidth = 640;
6325 const int kHeight = 360;
6326 const int kFpsLimit = 15;
6327 const int64_t kFrameIntervalMs = 150;
6328 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006330 DataRate::BitsPerSec(kTargetBitrateBps),
6331 DataRate::BitsPerSec(kTargetBitrateBps),
6332 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006333
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006334 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006335 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006336 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006337 video_stream_encoder_->SetSource(&source,
6338 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006339 timestamp_ms += kFrameIntervalMs;
6340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006341 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006342 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006343 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6345 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6346 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6347 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6348 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6349
6350 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006351 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006352 timestamp_ms += kFrameIntervalMs;
6353 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006354 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006355 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006356 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6357 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6358 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6359 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6360 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6361 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6362
6363 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006364 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006365 timestamp_ms += kFrameIntervalMs;
6366 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006367 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006368 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006369 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006370 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006371 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6372 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6373 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6374 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6375
Evan Shrubsole64469032020-06-11 10:45:29 +02006376 // Trigger cpu adapt up, expect no change because quality is most limited.
6377 {
6378 auto previous_sink_wants = source.sink_wants();
6379 // Store current sink wants since we expect no change ind if there is no
6380 // change then last__wants() is not updated.
6381 video_stream_encoder_->TriggerCpuUnderuse();
6382 timestamp_ms += kFrameIntervalMs;
6383 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6384 WaitForEncodedFrame(timestamp_ms);
6385 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6386 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6388 }
6389
6390 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6391 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006392 timestamp_ms += kFrameIntervalMs;
6393 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006394 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006395 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006396 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6397 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6398 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006399 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6400 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6401 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006402
Evan Shrubsole64469032020-06-11 10:45:29 +02006403 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006404 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006405 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006406 timestamp_ms += kFrameIntervalMs;
6407 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006408 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006409 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006410 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6411 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6412 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6413 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6414 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006415 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006416
6417 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006418 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006420 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006421 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006422
mflodmancc3d4422017-08-03 08:27:51 -07006423 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006424}
6425
mflodmancc3d4422017-08-03 08:27:51 -07006426TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006427 const int kFrameWidth = 1920;
6428 const int kFrameHeight = 1080;
6429 // 3/4 of 1920.
6430 const int kAdaptedFrameWidth = 1440;
6431 // 3/4 of 1080 rounded down to multiple of 4.
6432 const int kAdaptedFrameHeight = 808;
6433 const int kFramerate = 24;
6434
Henrik Boström381d1092020-05-12 18:49:07 +02006435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006436 DataRate::BitsPerSec(kTargetBitrateBps),
6437 DataRate::BitsPerSec(kTargetBitrateBps),
6438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006439 // Trigger reconfigure encoder (without resetting the entire instance).
6440 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006441 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6442 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006443 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006444 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006445 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006446 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006447 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006448 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006449
6450 video_source_.set_adaptation_enabled(true);
6451
6452 video_source_.IncomingCapturedFrame(
6453 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006454 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006455
6456 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006457 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006458 video_source_.IncomingCapturedFrame(
6459 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006460 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006461
mflodmancc3d4422017-08-03 08:27:51 -07006462 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006463}
6464
mflodmancc3d4422017-08-03 08:27:51 -07006465TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006466 const int kFrameWidth = 1280;
6467 const int kFrameHeight = 720;
6468 const int kLowFps = 2;
6469 const int kHighFps = 30;
6470
Henrik Boström381d1092020-05-12 18:49:07 +02006471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006472 DataRate::BitsPerSec(kTargetBitrateBps),
6473 DataRate::BitsPerSec(kTargetBitrateBps),
6474 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006475
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006476 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006477 max_framerate_ = kLowFps;
6478
6479 // Insert 2 seconds of 2fps video.
6480 for (int i = 0; i < kLowFps * 2; ++i) {
6481 video_source_.IncomingCapturedFrame(
6482 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6483 WaitForEncodedFrame(timestamp_ms);
6484 timestamp_ms += 1000 / kLowFps;
6485 }
6486
6487 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006489 DataRate::BitsPerSec(kTargetBitrateBps),
6490 DataRate::BitsPerSec(kTargetBitrateBps),
6491 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006492 video_source_.IncomingCapturedFrame(
6493 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6494 WaitForEncodedFrame(timestamp_ms);
6495 timestamp_ms += 1000 / kLowFps;
6496
6497 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6498
6499 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006500 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006501 const int kFrameIntervalMs = 1000 / kHighFps;
6502 max_framerate_ = kHighFps;
6503 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6504 video_source_.IncomingCapturedFrame(
6505 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6506 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6507 // be dropped if the encoder hans't been updated with the new higher target
6508 // framerate yet, causing it to overshoot the target bitrate and then
6509 // suffering the wrath of the media optimizer.
6510 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6511 timestamp_ms += kFrameIntervalMs;
6512 }
6513
6514 // Don expect correct measurement just yet, but it should be higher than
6515 // before.
6516 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6517
mflodmancc3d4422017-08-03 08:27:51 -07006518 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006519}
6520
mflodmancc3d4422017-08-03 08:27:51 -07006521TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006522 const int kFrameWidth = 1280;
6523 const int kFrameHeight = 720;
6524 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006525 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006526 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006527 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006528
Henrik Boström381d1092020-05-12 18:49:07 +02006529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006530 DataRate::BitsPerSec(kTargetBitrateBps),
6531 DataRate::BitsPerSec(kTargetBitrateBps),
6532 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006533 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006534
6535 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006536 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006537 video_source_.IncomingCapturedFrame(
6538 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6539 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006540 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006541
6542 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6544 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6545 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006546
6547 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006548 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006549 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006550
Per Kjellanderdcef6412020-10-07 15:09:05 +02006551 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006552 video_source_.IncomingCapturedFrame(
6553 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6554 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006555 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006556
mflodmancc3d4422017-08-03 08:27:51 -07006557 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006558}
ilnik6b826ef2017-06-16 06:53:48 -07006559
Niels Möller4db138e2018-04-19 09:04:13 +02006560TEST_F(VideoStreamEncoderTest,
6561 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6562 const int kFrameWidth = 1280;
6563 const int kFrameHeight = 720;
6564 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006565 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006566 DataRate::BitsPerSec(kTargetBitrateBps),
6567 DataRate::BitsPerSec(kTargetBitrateBps),
6568 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006569 video_source_.IncomingCapturedFrame(
6570 CreateFrame(1, kFrameWidth, kFrameHeight));
6571 WaitForEncodedFrame(1);
6572 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6573 .low_encode_usage_threshold_percent,
6574 default_options.low_encode_usage_threshold_percent);
6575 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6576 .high_encode_usage_threshold_percent,
6577 default_options.high_encode_usage_threshold_percent);
6578 video_stream_encoder_->Stop();
6579}
6580
6581TEST_F(VideoStreamEncoderTest,
6582 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6583 const int kFrameWidth = 1280;
6584 const int kFrameHeight = 720;
6585 CpuOveruseOptions hardware_options;
6586 hardware_options.low_encode_usage_threshold_percent = 150;
6587 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006588 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006589
Henrik Boström381d1092020-05-12 18:49:07 +02006590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006591 DataRate::BitsPerSec(kTargetBitrateBps),
6592 DataRate::BitsPerSec(kTargetBitrateBps),
6593 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006594 video_source_.IncomingCapturedFrame(
6595 CreateFrame(1, kFrameWidth, kFrameHeight));
6596 WaitForEncodedFrame(1);
6597 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6598 .low_encode_usage_threshold_percent,
6599 hardware_options.low_encode_usage_threshold_percent);
6600 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6601 .high_encode_usage_threshold_percent,
6602 hardware_options.high_encode_usage_threshold_percent);
6603 video_stream_encoder_->Stop();
6604}
6605
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006606TEST_F(VideoStreamEncoderTest,
6607 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6608 const int kFrameWidth = 1280;
6609 const int kFrameHeight = 720;
6610
6611 const CpuOveruseOptions default_options;
6612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6613 DataRate::BitsPerSec(kTargetBitrateBps),
6614 DataRate::BitsPerSec(kTargetBitrateBps),
6615 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6616 video_source_.IncomingCapturedFrame(
6617 CreateFrame(1, kFrameWidth, kFrameHeight));
6618 WaitForEncodedFrame(1);
6619 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6620 .low_encode_usage_threshold_percent,
6621 default_options.low_encode_usage_threshold_percent);
6622 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6623 .high_encode_usage_threshold_percent,
6624 default_options.high_encode_usage_threshold_percent);
6625
6626 CpuOveruseOptions hardware_options;
6627 hardware_options.low_encode_usage_threshold_percent = 150;
6628 hardware_options.high_encode_usage_threshold_percent = 200;
6629 fake_encoder_.SetIsHardwareAccelerated(true);
6630
6631 video_source_.IncomingCapturedFrame(
6632 CreateFrame(2, kFrameWidth, kFrameHeight));
6633 WaitForEncodedFrame(2);
6634
6635 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6636 .low_encode_usage_threshold_percent,
6637 hardware_options.low_encode_usage_threshold_percent);
6638 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6639 .high_encode_usage_threshold_percent,
6640 hardware_options.high_encode_usage_threshold_percent);
6641
6642 video_stream_encoder_->Stop();
6643}
6644
Niels Möller6bb5ab92019-01-11 11:11:10 +01006645TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6646 const int kFrameWidth = 320;
6647 const int kFrameHeight = 240;
6648 const int kFps = 30;
6649 const int kTargetBitrateBps = 120000;
6650 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6651
Henrik Boström381d1092020-05-12 18:49:07 +02006652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006653 DataRate::BitsPerSec(kTargetBitrateBps),
6654 DataRate::BitsPerSec(kTargetBitrateBps),
6655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006656
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006657 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006658 max_framerate_ = kFps;
6659
6660 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6661 fake_encoder_.SimulateOvershoot(1.0);
6662 int num_dropped = 0;
6663 for (int i = 0; i < kNumFramesInRun; ++i) {
6664 video_source_.IncomingCapturedFrame(
6665 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6666 // Wait up to two frame durations for a frame to arrive.
6667 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6668 ++num_dropped;
6669 }
6670 timestamp_ms += 1000 / kFps;
6671 }
6672
Erik Språnga8d48ab2019-02-08 14:17:40 +01006673 // Framerate should be measured to be near the expected target rate.
6674 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6675
6676 // Frame drops should be within 5% of expected 0%.
6677 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006678
6679 // Make encoder produce frames at double the expected bitrate during 3 seconds
6680 // of video, verify number of drops. Rate needs to be slightly changed in
6681 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006682 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006683 const RateControlSettings trials =
6684 RateControlSettings::ParseFromFieldTrials();
6685 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006686 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006687 // frame dropping since the adjuter will try to just lower the target
6688 // bitrate rather than drop frames. If network headroom can be used, it
6689 // doesn't push back as hard so we don't need quite as much overshoot.
6690 // These numbers are unfortunately a bit magical but there's not trivial
6691 // way to algebraically infer them.
6692 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6693 overshoot_factor = 2.4;
6694 } else {
6695 overshoot_factor = 4.0;
6696 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006697 }
6698 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006700 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6701 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6702 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006703 num_dropped = 0;
6704 for (int i = 0; i < kNumFramesInRun; ++i) {
6705 video_source_.IncomingCapturedFrame(
6706 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6707 // Wait up to two frame durations for a frame to arrive.
6708 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6709 ++num_dropped;
6710 }
6711 timestamp_ms += 1000 / kFps;
6712 }
6713
Henrik Boström381d1092020-05-12 18:49:07 +02006714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006715 DataRate::BitsPerSec(kTargetBitrateBps),
6716 DataRate::BitsPerSec(kTargetBitrateBps),
6717 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006718
6719 // Target framerate should be still be near the expected target, despite
6720 // the frame drops.
6721 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6722
6723 // Frame drops should be within 5% of expected 50%.
6724 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006725
6726 video_stream_encoder_->Stop();
6727}
6728
6729TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6730 const int kFrameWidth = 320;
6731 const int kFrameHeight = 240;
6732 const int kActualInputFps = 24;
6733 const int kTargetBitrateBps = 120000;
6734
6735 ASSERT_GT(max_framerate_, kActualInputFps);
6736
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006737 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006738 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006740 DataRate::BitsPerSec(kTargetBitrateBps),
6741 DataRate::BitsPerSec(kTargetBitrateBps),
6742 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006743
6744 // Insert 3 seconds of video, with an input fps lower than configured max.
6745 for (int i = 0; i < kActualInputFps * 3; ++i) {
6746 video_source_.IncomingCapturedFrame(
6747 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6748 // Wait up to two frame durations for a frame to arrive.
6749 WaitForEncodedFrame(timestamp_ms);
6750 timestamp_ms += 1000 / kActualInputFps;
6751 }
6752
6753 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6754
6755 video_stream_encoder_->Stop();
6756}
6757
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006758TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006759 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006760 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006761 DataRate::BitsPerSec(kTargetBitrateBps),
6762 DataRate::BitsPerSec(kTargetBitrateBps),
6763 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006764
6765 fake_encoder_.BlockNextEncode();
6766 video_source_.IncomingCapturedFrame(
6767 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6768 WaitForEncodedFrame(1);
6769 // On the very first frame full update should be forced.
6770 rect = fake_encoder_.GetLastUpdateRect();
6771 EXPECT_EQ(rect.offset_x, 0);
6772 EXPECT_EQ(rect.offset_y, 0);
6773 EXPECT_EQ(rect.height, codec_height_);
6774 EXPECT_EQ(rect.width, codec_width_);
6775 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6776 // call to ContinueEncode.
6777 video_source_.IncomingCapturedFrame(
6778 CreateFrameWithUpdatedPixel(2, nullptr, 1));
6779 ExpectDroppedFrame();
6780 video_source_.IncomingCapturedFrame(
6781 CreateFrameWithUpdatedPixel(3, nullptr, 10));
6782 ExpectDroppedFrame();
6783 fake_encoder_.ContinueEncode();
6784 WaitForEncodedFrame(3);
6785 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
6786 rect = fake_encoder_.GetLastUpdateRect();
6787 EXPECT_EQ(rect.offset_x, 1);
6788 EXPECT_EQ(rect.offset_y, 0);
6789 EXPECT_EQ(rect.width, 10);
6790 EXPECT_EQ(rect.height, 1);
6791
6792 video_source_.IncomingCapturedFrame(
6793 CreateFrameWithUpdatedPixel(4, nullptr, 0));
6794 WaitForEncodedFrame(4);
6795 // Previous frame was encoded, so no accumulation should happen.
6796 rect = fake_encoder_.GetLastUpdateRect();
6797 EXPECT_EQ(rect.offset_x, 0);
6798 EXPECT_EQ(rect.offset_y, 0);
6799 EXPECT_EQ(rect.width, 1);
6800 EXPECT_EQ(rect.height, 1);
6801
6802 video_stream_encoder_->Stop();
6803}
6804
Erik Språngd7329ca2019-02-21 21:19:53 +01006805TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02006806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006807 DataRate::BitsPerSec(kTargetBitrateBps),
6808 DataRate::BitsPerSec(kTargetBitrateBps),
6809 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006810
6811 // First frame is always keyframe.
6812 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6813 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01006814 EXPECT_THAT(
6815 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006816 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006817
6818 // Insert delta frame.
6819 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6820 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01006821 EXPECT_THAT(
6822 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006823 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006824
6825 // Request next frame be a key-frame.
6826 video_stream_encoder_->SendKeyFrame();
6827 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6828 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01006829 EXPECT_THAT(
6830 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006831 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006832
6833 video_stream_encoder_->Stop();
6834}
6835
6836TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
6837 // Setup simulcast with three streams.
6838 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006840 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6841 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6842 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006843 // Wait for all three layers before triggering event.
6844 sink_.SetNumExpectedLayers(3);
6845
6846 // First frame is always keyframe.
6847 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6848 WaitForEncodedFrame(1);
6849 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006850 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6851 VideoFrameType::kVideoFrameKey,
6852 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006853
6854 // Insert delta frame.
6855 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6856 WaitForEncodedFrame(2);
6857 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006858 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6859 VideoFrameType::kVideoFrameDelta,
6860 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006861
6862 // Request next frame be a key-frame.
6863 // Only first stream is configured to produce key-frame.
6864 video_stream_encoder_->SendKeyFrame();
6865 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6866 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02006867
6868 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
6869 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01006870 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006871 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02006872 VideoFrameType::kVideoFrameKey,
6873 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006874
6875 video_stream_encoder_->Stop();
6876}
6877
6878TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
6879 // Configure internal source factory and setup test again.
6880 encoder_factory_.SetHasInternalSource(true);
6881 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006882 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006883 DataRate::BitsPerSec(kTargetBitrateBps),
6884 DataRate::BitsPerSec(kTargetBitrateBps),
6885 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006886
6887 // Call encoder directly, simulating internal source where encoded frame
6888 // callback in VideoStreamEncoder is called despite no OnFrame().
6889 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
6890 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006891 EXPECT_THAT(
6892 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006893 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006894
Niels Möller8f7ce222019-03-21 15:43:58 +01006895 const std::vector<VideoFrameType> kDeltaFrame = {
6896 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01006897 // Need to set timestamp manually since manually for injected frame.
6898 VideoFrame frame = CreateFrame(101, nullptr);
6899 frame.set_timestamp(101);
6900 fake_encoder_.InjectFrame(frame, false);
6901 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006902 EXPECT_THAT(
6903 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006904 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006905
6906 // Request key-frame. The forces a dummy frame down into the encoder.
6907 fake_encoder_.ExpectNullFrame();
6908 video_stream_encoder_->SendKeyFrame();
6909 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006910 EXPECT_THAT(
6911 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006912 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006913
6914 video_stream_encoder_->Stop();
6915}
Erik Språngb7cb7b52019-02-26 15:52:33 +01006916
6917TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
6918 // Configure internal source factory and setup test again.
6919 encoder_factory_.SetHasInternalSource(true);
6920 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006921 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006922 DataRate::BitsPerSec(kTargetBitrateBps),
6923 DataRate::BitsPerSec(kTargetBitrateBps),
6924 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006925
6926 int64_t timestamp = 1;
6927 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02006928 image.SetEncodedData(
6929 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01006930 image.capture_time_ms_ = ++timestamp;
6931 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
6932 const int64_t kEncodeFinishDelayMs = 10;
6933 image.timing_.encode_start_ms = timestamp;
6934 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
6935 fake_encoder_.InjectEncodedImage(image);
6936 // Wait for frame without incrementing clock.
6937 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
6938 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
6939 // capture timestamp should be kEncodeFinishDelayMs in the past.
6940 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006941 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006942
6943 video_stream_encoder_->Stop();
6944}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006945
6946TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006947 // SPS contains VUI with restrictions on the maximum number of reordered
6948 // pictures, there is no need to rewrite the bitstream to enable faster
6949 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006950 ResetEncoder("H264", 1, 1, 1, false);
6951
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006952 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6953 DataRate::BitsPerSec(kTargetBitrateBps),
6954 DataRate::BitsPerSec(kTargetBitrateBps),
6955 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6956 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006957
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006958 fake_encoder_.SetEncodedImageData(
6959 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006960
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006961 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6962 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006963
6964 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6965 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006966
6967 video_stream_encoder_->Stop();
6968}
6969
6970TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006971 // SPS does not contain VUI, the bitstream is will be rewritten with added
6972 // VUI with restrictions on the maximum number of reordered pictures to
6973 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006974 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
6975 0x00, 0x00, 0x03, 0x03, 0xF4,
6976 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006977 ResetEncoder("H264", 1, 1, 1, false);
6978
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6980 DataRate::BitsPerSec(kTargetBitrateBps),
6981 DataRate::BitsPerSec(kTargetBitrateBps),
6982 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6983 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006984
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006985 fake_encoder_.SetEncodedImageData(
6986 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006987
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006988 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6989 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006990
6991 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6992 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006993
6994 video_stream_encoder_->Stop();
6995}
6996
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006997TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
6998 const int kFrameWidth = 1280;
6999 const int kFrameHeight = 720;
7000 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7001
Henrik Boström381d1092020-05-12 18:49:07 +02007002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007003 DataRate::BitsPerSec(kTargetBitrateBps),
7004 DataRate::BitsPerSec(kTargetBitrateBps),
7005 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007006 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7007
7008 // Insert a first video frame. It should be dropped because of downscale in
7009 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007010 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007011 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7012 frame.set_rotation(kVideoRotation_270);
7013 video_source_.IncomingCapturedFrame(frame);
7014
7015 ExpectDroppedFrame();
7016
7017 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007018 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007019 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7020 frame.set_rotation(kVideoRotation_90);
7021 video_source_.IncomingCapturedFrame(frame);
7022
7023 WaitForEncodedFrame(timestamp_ms);
7024 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7025
7026 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007027 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007028 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7029 frame.set_rotation(kVideoRotation_180);
7030 video_source_.IncomingCapturedFrame(frame);
7031
7032 WaitForEncodedFrame(timestamp_ms);
7033 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7034
7035 video_stream_encoder_->Stop();
7036}
7037
Erik Språng5056af02019-09-02 15:53:11 +02007038TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7039 const int kFrameWidth = 320;
7040 const int kFrameHeight = 180;
7041
7042 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007044 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7045 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7046 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007047 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007048 /*rtt_ms=*/0,
7049 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007050
7051 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007052 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007053 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7054 frame.set_rotation(kVideoRotation_270);
7055 video_source_.IncomingCapturedFrame(frame);
7056 WaitForEncodedFrame(timestamp_ms);
7057
7058 // Set a target rate below the minimum allowed by the codec settings.
7059 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007060 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7061 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007063 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007064 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007065 /*link_allocation=*/target_rate,
7066 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007067 /*rtt_ms=*/0,
7068 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007069 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7070
7071 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7072 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7073 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007074 DataRate allocation_sum =
7075 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007076 EXPECT_EQ(min_rate, allocation_sum);
7077 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7078
7079 video_stream_encoder_->Stop();
7080}
7081
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007082TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007083 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007084 DataRate::BitsPerSec(kTargetBitrateBps),
7085 DataRate::BitsPerSec(kTargetBitrateBps),
7086 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007087 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007088 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007089 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7090 WaitForEncodedFrame(1);
7091
7092 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7093 ASSERT_TRUE(prev_rate_settings.has_value());
7094 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7095 kDefaultFramerate);
7096
7097 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7098 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7099 timestamp_ms += 1000 / kDefaultFramerate;
7100 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7101 WaitForEncodedFrame(timestamp_ms);
7102 }
7103 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7104 kDefaultFramerate);
7105 // Capture larger frame to trigger a reconfigure.
7106 codec_height_ *= 2;
7107 codec_width_ *= 2;
7108 timestamp_ms += 1000 / kDefaultFramerate;
7109 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7110 WaitForEncodedFrame(timestamp_ms);
7111
7112 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7113 auto current_rate_settings =
7114 fake_encoder_.GetAndResetLastRateControlSettings();
7115 // Ensure we have actually reconfigured twice
7116 // The rate settings should have been set again even though
7117 // they haven't changed.
7118 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007119 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007120
7121 video_stream_encoder_->Stop();
7122}
7123
philipeld9cc8c02019-09-16 14:53:40 +02007124struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007125 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7126 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7127 MOCK_METHOD(void,
7128 RequestEncoderSwitch,
7129 (const webrtc::SdpVideoFormat& format),
7130 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007131};
7132
7133TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
7134 constexpr int kDontCare = 100;
7135
7136 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7137 video_send_config_.encoder_settings.encoder_switch_request_callback =
7138 &switch_callback;
7139 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7140 encoder_config.codec_type = kVideoCodecVP8;
7141 webrtc::test::ScopedFieldTrials field_trial(
7142 "WebRTC-NetworkCondition-EncoderSwitch/"
7143 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
7144 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7145
7146 // Reset encoder for new configuration to take effect.
7147 ConfigureEncoder(std::move(encoder_config));
7148
7149 // Send one frame to trigger ReconfigureEncoder.
7150 video_source_.IncomingCapturedFrame(
7151 CreateFrame(kDontCare, kDontCare, kDontCare));
7152
7153 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01007154 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
7155 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02007156 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01007157 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02007158
Henrik Boström381d1092020-05-12 18:49:07 +02007159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007160 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7161 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7162 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02007163 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007164 /*rtt_ms=*/0,
7165 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007166 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02007167
7168 video_stream_encoder_->Stop();
7169}
7170
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01007171TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
7172 constexpr int kDontCare = 100;
7173
7174 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7175 video_send_config_.encoder_settings.encoder_switch_request_callback =
7176 &switch_callback;
7177 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7178 encoder_config.codec_type = kVideoCodecVP8;
7179 webrtc::test::ScopedFieldTrials field_trial(
7180 "WebRTC-NetworkCondition-EncoderSwitch/"
7181 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
7182 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7183
7184 // Reset encoder for new configuration to take effect.
7185 ConfigureEncoder(std::move(encoder_config));
7186
7187 // Send one frame to trigger ReconfigureEncoder.
7188 video_source_.IncomingCapturedFrame(
7189 CreateFrame(kDontCare, kDontCare, kDontCare));
7190
7191 using Config = EncoderSwitchRequestCallback::Config;
7192 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
7193 .Times(0);
7194
Henrik Boström381d1092020-05-12 18:49:07 +02007195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01007196 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
7197 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
7198 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
7199 /*fraction_lost=*/0,
7200 /*rtt_ms=*/0,
7201 /*cwnd_reduce_ratio=*/0);
7202
7203 video_stream_encoder_->Stop();
7204}
7205
philipeld9cc8c02019-09-16 14:53:40 +02007206TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
7207 constexpr int kSufficientBitrateToNotDrop = 1000;
7208 constexpr int kHighRes = 500;
7209 constexpr int kLowRes = 100;
7210
7211 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7212 video_send_config_.encoder_settings.encoder_switch_request_callback =
7213 &switch_callback;
7214 webrtc::test::ScopedFieldTrials field_trial(
7215 "WebRTC-NetworkCondition-EncoderSwitch/"
7216 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
7217 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7218 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7219 encoder_config.codec_type = kVideoCodecH264;
7220
7221 // Reset encoder for new configuration to take effect.
7222 ConfigureEncoder(std::move(encoder_config));
7223
7224 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7225 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7226 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007227 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007228 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7229 /*stable_target_bitrate=*/
7230 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7231 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02007232 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007233 /*rtt_ms=*/0,
7234 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02007235
7236 // Send one frame to trigger ReconfigureEncoder.
7237 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
7238 WaitForEncodedFrame(1);
7239
7240 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01007241 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
7242 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02007243 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01007244 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02007245
7246 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
7247 WaitForEncodedFrame(2);
7248
7249 video_stream_encoder_->Stop();
7250}
7251
philipel9b058032020-02-10 11:30:00 +01007252TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7253 constexpr int kDontCare = 100;
7254 StrictMock<MockEncoderSelector> encoder_selector;
7255 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7256 &fake_encoder_, &encoder_selector);
7257 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7258
7259 // Reset encoder for new configuration to take effect.
7260 ConfigureEncoder(video_encoder_config_.Copy());
7261
7262 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7263
7264 video_source_.IncomingCapturedFrame(
7265 CreateFrame(kDontCare, kDontCare, kDontCare));
7266 video_stream_encoder_->Stop();
7267
7268 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7269 // to it's factory, so in order for the encoder instance in the
7270 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7271 // reset the |video_stream_encoder_| here.
7272 video_stream_encoder_.reset();
7273}
7274
7275TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7276 constexpr int kDontCare = 100;
7277
7278 NiceMock<MockEncoderSelector> encoder_selector;
7279 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7280 video_send_config_.encoder_settings.encoder_switch_request_callback =
7281 &switch_callback;
7282 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7283 &fake_encoder_, &encoder_selector);
7284 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7285
7286 // Reset encoder for new configuration to take effect.
7287 ConfigureEncoder(video_encoder_config_.Copy());
7288
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007289 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007290 .WillByDefault(Return(SdpVideoFormat("AV1")));
7291 EXPECT_CALL(switch_callback,
7292 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7293 Field(&SdpVideoFormat::name, "AV1"))));
7294
Henrik Boström381d1092020-05-12 18:49:07 +02007295 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007296 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7297 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7298 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007299 /*fraction_lost=*/0,
7300 /*rtt_ms=*/0,
7301 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007302 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007303
7304 video_stream_encoder_->Stop();
7305}
7306
7307TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7308 constexpr int kSufficientBitrateToNotDrop = 1000;
7309 constexpr int kDontCare = 100;
7310
7311 NiceMock<MockVideoEncoder> video_encoder;
7312 NiceMock<MockEncoderSelector> encoder_selector;
7313 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7314 video_send_config_.encoder_settings.encoder_switch_request_callback =
7315 &switch_callback;
7316 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7317 &video_encoder, &encoder_selector);
7318 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7319
7320 // Reset encoder for new configuration to take effect.
7321 ConfigureEncoder(video_encoder_config_.Copy());
7322
7323 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7324 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7325 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007327 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7328 /*stable_target_bitrate=*/
7329 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7330 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007331 /*fraction_lost=*/0,
7332 /*rtt_ms=*/0,
7333 /*cwnd_reduce_ratio=*/0);
7334
7335 ON_CALL(video_encoder, Encode(_, _))
7336 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7337 ON_CALL(encoder_selector, OnEncoderBroken())
7338 .WillByDefault(Return(SdpVideoFormat("AV2")));
7339
7340 rtc::Event encode_attempted;
7341 EXPECT_CALL(switch_callback,
7342 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7343 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7344 EXPECT_EQ(format.name, "AV2");
7345 encode_attempted.Set();
7346 });
7347
7348 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7349 encode_attempted.Wait(3000);
7350
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007351 AdvanceTime(TimeDelta::Millis(0));
7352
philipel9b058032020-02-10 11:30:00 +01007353 video_stream_encoder_->Stop();
7354
7355 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7356 // to it's factory, so in order for the encoder instance in the
7357 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7358 // reset the |video_stream_encoder_| here.
7359 video_stream_encoder_.reset();
7360}
7361
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007362TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007363 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007364 const int kFrameWidth = 320;
7365 const int kFrameHeight = 180;
7366
7367 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007368 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007370 /*target_bitrate=*/rate,
7371 /*stable_target_bitrate=*/rate,
7372 /*link_allocation=*/rate,
7373 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007374 /*rtt_ms=*/0,
7375 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007376
7377 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007378 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007379 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7380 frame.set_rotation(kVideoRotation_270);
7381 video_source_.IncomingCapturedFrame(frame);
7382 WaitForEncodedFrame(timestamp_ms);
7383 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7384
7385 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007386 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007387 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007388 /*target_bitrate=*/new_stable_rate,
7389 /*stable_target_bitrate=*/new_stable_rate,
7390 /*link_allocation=*/rate,
7391 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007392 /*rtt_ms=*/0,
7393 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007394 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7395 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7396 video_stream_encoder_->Stop();
7397}
7398
7399TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007400 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007401 const int kFrameWidth = 320;
7402 const int kFrameHeight = 180;
7403
7404 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007405 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007407 /*target_bitrate=*/rate,
7408 /*stable_target_bitrate=*/rate,
7409 /*link_allocation=*/rate,
7410 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007411 /*rtt_ms=*/0,
7412 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007413
7414 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007415 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007416 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7417 frame.set_rotation(kVideoRotation_270);
7418 video_source_.IncomingCapturedFrame(frame);
7419 WaitForEncodedFrame(timestamp_ms);
7420 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7421
7422 // Set a higher target rate without changing the link_allocation. Should not
7423 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007424 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007425 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007426 /*target_bitrate=*/rate,
7427 /*stable_target_bitrate=*/new_stable_rate,
7428 /*link_allocation=*/rate,
7429 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007430 /*rtt_ms=*/0,
7431 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007432 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7433 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7434 video_stream_encoder_->Stop();
7435}
7436
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007437TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7438 test::ScopedFieldTrials field_trials(
7439 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7440 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7441 const int kFramerateFps = 30;
7442 const int kWidth = 1920;
7443 const int kHeight = 1080;
7444 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7445 // Works on screenshare mode.
7446 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7447 // We rely on the automatic resolution adaptation, but we handle framerate
7448 // adaptation manually by mocking the stats proxy.
7449 video_source_.set_adaptation_enabled(true);
7450
7451 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007453 DataRate::BitsPerSec(kTargetBitrateBps),
7454 DataRate::BitsPerSec(kTargetBitrateBps),
7455 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007456 video_stream_encoder_->SetSource(&video_source_,
7457 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007458 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007459
7460 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7461 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7462
7463 // Pass enough frames with the full update to trigger animation detection.
7464 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007465 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007466 frame.set_ntp_time_ms(timestamp_ms);
7467 frame.set_timestamp_us(timestamp_ms * 1000);
7468 video_source_.IncomingCapturedFrame(frame);
7469 WaitForEncodedFrame(timestamp_ms);
7470 }
7471
7472 // Resolution should be limited.
7473 rtc::VideoSinkWants expected;
7474 expected.max_framerate_fps = kFramerateFps;
7475 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007476 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007477
7478 // Pass one frame with no known update.
7479 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007480 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007481 frame.set_ntp_time_ms(timestamp_ms);
7482 frame.set_timestamp_us(timestamp_ms * 1000);
7483 frame.clear_update_rect();
7484
7485 video_source_.IncomingCapturedFrame(frame);
7486 WaitForEncodedFrame(timestamp_ms);
7487
7488 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007489 EXPECT_THAT(video_source_.sink_wants(),
7490 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007491
7492 video_stream_encoder_->Stop();
7493}
7494
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007495TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7496 const int kWidth = 720; // 540p adapted down.
7497 const int kHeight = 405;
7498 const int kNumFrames = 3;
7499 // Works on screenshare mode.
7500 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7501 /*num_spatial_layers=*/2, /*screenshare=*/true);
7502
7503 video_source_.set_adaptation_enabled(true);
7504
7505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7506 DataRate::BitsPerSec(kTargetBitrateBps),
7507 DataRate::BitsPerSec(kTargetBitrateBps),
7508 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7509
7510 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7511
7512 // Pass enough frames with the full update to trigger animation detection.
7513 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007514 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007515 frame.set_ntp_time_ms(timestamp_ms);
7516 frame.set_timestamp_us(timestamp_ms * 1000);
7517 video_source_.IncomingCapturedFrame(frame);
7518 WaitForEncodedFrame(timestamp_ms);
7519 }
7520
7521 video_stream_encoder_->Stop();
7522}
7523
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007524TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7525 const float downscale_factors[] = {4.0, 2.0, 1.0};
7526 const int number_layers =
7527 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7528 VideoEncoderConfig config;
7529 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7530 for (int i = 0; i < number_layers; ++i) {
7531 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7532 config.simulcast_layers[i].active = true;
7533 }
7534 config.video_stream_factory =
7535 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7536 "VP8", /*max qp*/ 56, /*screencast*/ false,
7537 /*screenshare enabled*/ false);
7538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7539 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7540 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7541 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7542
7543 // First initialization.
7544 // Encoder should be initialized. Next frame should be key frame.
7545 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7546 sink_.SetNumExpectedLayers(number_layers);
7547 int64_t timestamp_ms = kFrameIntervalMs;
7548 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7549 WaitForEncodedFrame(timestamp_ms);
7550 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7551 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7552 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7553 VideoFrameType::kVideoFrameKey,
7554 VideoFrameType::kVideoFrameKey}));
7555
7556 // Disable top layer.
7557 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7558 config.simulcast_layers[number_layers - 1].active = false;
7559 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7560 sink_.SetNumExpectedLayers(number_layers - 1);
7561 timestamp_ms += kFrameIntervalMs;
7562 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7563 WaitForEncodedFrame(timestamp_ms);
7564 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7565 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7566 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7567 VideoFrameType::kVideoFrameDelta,
7568 VideoFrameType::kVideoFrameDelta}));
7569
7570 // Re-enable top layer.
7571 // Encoder should be re-initialized. Next frame should be key frame.
7572 config.simulcast_layers[number_layers - 1].active = true;
7573 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7574 sink_.SetNumExpectedLayers(number_layers);
7575 timestamp_ms += kFrameIntervalMs;
7576 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7577 WaitForEncodedFrame(timestamp_ms);
7578 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7579 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7580 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7581 VideoFrameType::kVideoFrameKey,
7582 VideoFrameType::kVideoFrameKey}));
7583
7584 // Top layer max rate change.
7585 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7586 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7587 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7588 sink_.SetNumExpectedLayers(number_layers);
7589 timestamp_ms += kFrameIntervalMs;
7590 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7591 WaitForEncodedFrame(timestamp_ms);
7592 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7593 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7594 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7595 VideoFrameType::kVideoFrameDelta,
7596 VideoFrameType::kVideoFrameDelta}));
7597
7598 // Top layer resolution change.
7599 // Encoder should be re-initialized. Next frame should be key frame.
7600 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7601 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7602 sink_.SetNumExpectedLayers(number_layers);
7603 timestamp_ms += kFrameIntervalMs;
7604 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7605 WaitForEncodedFrame(timestamp_ms);
7606 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7607 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7608 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7609 VideoFrameType::kVideoFrameKey,
7610 VideoFrameType::kVideoFrameKey}));
7611 video_stream_encoder_->Stop();
7612}
7613
perkj26091b12016-09-01 01:17:40 -07007614} // namespace webrtc