blob: aed619db26106dd278c2589f18872215da010d3c [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020040#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020043#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010044#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020045#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020060using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020062using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020063using ::testing::Ge;
64using ::testing::Gt;
65using ::testing::Le;
66using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010067using ::testing::Matcher;
68using ::testing::NiceMock;
69using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010070using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020071using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080072
perkj803d97f2016-11-01 11:45:46 -070073namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010075const int kQpLow = 1;
76const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020077const int kMinFramerateFps = 2;
78const int kMinBalancedFramerateFps = 7;
79const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080080const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020082const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010083const uint32_t kSimulcastTargetBitrateBps = 3150000;
84const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080085const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070086const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020088const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020089const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
91const VideoEncoder::ResolutionBitrateLimits
92 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020094uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
95 0x00, 0x00, 0x03, 0x03, 0xF4,
96 0x05, 0x03, 0xC7, 0xE0, 0x1B,
97 0x41, 0x10, 0x8D, 0x00};
98
perkj803d97f2016-11-01 11:45:46 -070099class TestBuffer : public webrtc::I420Buffer {
100 public:
101 TestBuffer(rtc::Event* event, int width, int height)
102 : I420Buffer(width, height), event_(event) {}
103
104 private:
105 friend class rtc::RefCountedObject<TestBuffer>;
106 ~TestBuffer() override {
107 if (event_)
108 event_->Set();
109 }
110 rtc::Event* const event_;
111};
112
Noah Richards51db4212019-06-12 06:59:12 -0700113// A fake native buffer that can't be converted to I420.
114class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
115 public:
116 FakeNativeBuffer(rtc::Event* event, int width, int height)
117 : event_(event), width_(width), height_(height) {}
118 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
119 int width() const override { return width_; }
120 int height() const override { return height_; }
121 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
122 return nullptr;
123 }
124
125 private:
126 friend class rtc::RefCountedObject<FakeNativeBuffer>;
127 ~FakeNativeBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132 const int width_;
133 const int height_;
134};
135
Evan Shrubsole895556e2020-10-05 09:15:13 +0200136// A fake native buffer that is backed by an NV12 buffer.
137class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
140 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
141
142 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
143 int width() const override { return nv12_buffer_->width(); }
144 int height() const override { return nv12_buffer_->height(); }
145 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
146 return nv12_buffer_->ToI420();
147 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200148 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
149 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
150 if (absl::c_find(types, Type::kNV12) != types.end()) {
151 return nv12_buffer_;
152 }
153 return nullptr;
154 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200155 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
156
157 private:
158 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
159 ~FakeNV12NativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
164 rtc::Event* const event_;
165};
166
Niels Möller7dc26b72017-12-06 10:27:48 +0100167class CpuOveruseDetectorProxy : public OveruseFrameDetector {
168 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200169 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
170 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200171 last_target_framerate_fps_(-1),
172 framerate_updated_event_(true /* manual_reset */,
173 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100174 virtual ~CpuOveruseDetectorProxy() {}
175
176 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200177 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100178 last_target_framerate_fps_ = framerate_fps;
179 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200180 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100181 }
182
183 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200184 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100185 return last_target_framerate_fps_;
186 }
187
Niels Möller4db138e2018-04-19 09:04:13 +0200188 CpuOveruseOptions GetOptions() { return options_; }
189
Henrik Boström381d1092020-05-12 18:49:07 +0200190 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
191
Niels Möller7dc26b72017-12-06 10:27:48 +0100192 private:
Markus Handella3765182020-07-08 13:13:32 +0200193 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100194 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200195 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100196};
197
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200198class FakeVideoSourceRestrictionsListener
199 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200200 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200201 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200202 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200203 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200204 RTC_DCHECK(was_restrictions_updated_);
205 }
206
207 rtc::Event* restrictions_updated_event() {
208 return &restrictions_updated_event_;
209 }
210
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200211 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200212 void OnVideoSourceRestrictionsUpdated(
213 VideoSourceRestrictions restrictions,
214 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200215 rtc::scoped_refptr<Resource> reason,
216 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200217 was_restrictions_updated_ = true;
218 restrictions_updated_event_.Set();
219 }
220
221 private:
222 bool was_restrictions_updated_;
223 rtc::Event restrictions_updated_event_;
224};
225
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200226auto WantsFps(Matcher<int> fps_matcher) {
227 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
228 fps_matcher);
229}
230
231auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
232 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
233 AllOf(max_pixel_matcher, Gt(0)));
234}
235
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200236auto ResolutionMax() {
237 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
240 Eq(absl::nullopt)));
241}
242
243auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245}
246
247auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200249}
250
251auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200253}
254
255auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200256 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200257}
258
259auto FpsMaxResolutionMax() {
260 return AllOf(FpsMax(), ResolutionMax());
261}
262
263auto UnlimitedSinkWants() {
264 return AllOf(FpsUnlimited(), ResolutionMax());
265}
266
267auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
268 Matcher<int> fps_range_matcher;
269
270 if (last_frame_pixels <= 320 * 240) {
271 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200272 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200273 fps_range_matcher = AllOf(Ge(10), Le(15));
274 } else if (last_frame_pixels <= 640 * 480) {
275 fps_range_matcher = Ge(15);
276 } else {
277 fps_range_matcher = Eq(kDefaultFramerate);
278 }
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_range_matcher);
281}
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
284 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
285 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
286}
287
288auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
289 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
290}
291
292auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
293 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
294}
295
296auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
299}
300
301auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
304}
305
306auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
307 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
308 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
309}
310
311auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
312 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
313 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
314}
315
mflodmancc3d4422017-08-03 08:27:51 -0700316class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700317 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200318 VideoStreamEncoderUnderTest(TimeController* time_controller,
319 TaskQueueFactory* task_queue_factory,
320 SendStatisticsProxy* stats_proxy,
321 const VideoStreamEncoderSettings& settings)
322 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100323 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200324 stats_proxy,
325 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200326 std::unique_ptr<OveruseFrameDetector>(
327 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100328 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100329 task_queue_factory),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200330 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200331 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200332 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200333 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200334 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200335 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200336 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200337 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100338 }
perkj803d97f2016-11-01 11:45:46 -0700339
Henrik Boström381d1092020-05-12 18:49:07 +0200340 void SetSourceAndWaitForRestrictionsUpdated(
341 rtc::VideoSourceInterface<VideoFrame>* source,
342 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200343 FakeVideoSourceRestrictionsListener listener;
344 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200345 SetSource(source, degradation_preference);
346 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200347 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200348 }
349
350 void SetSourceAndWaitForFramerateUpdated(
351 rtc::VideoSourceInterface<VideoFrame>* source,
352 const DegradationPreference& degradation_preference) {
353 overuse_detector_proxy_->framerate_updated_event()->Reset();
354 SetSource(source, degradation_preference);
355 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
356 }
357
358 void OnBitrateUpdatedAndWaitForManagedResources(
359 DataRate target_bitrate,
360 DataRate stable_target_bitrate,
361 DataRate link_allocation,
362 uint8_t fraction_lost,
363 int64_t round_trip_time_ms,
364 double cwnd_reduce_ratio) {
365 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
366 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
367 // Bitrate is updated on the encoder queue.
368 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200369 }
370
kthelgason2fc52542017-03-03 00:24:41 -0800371 // This is used as a synchronisation mechanism, to make sure that the
372 // encoder queue is not blocked before we start sending it frames.
373 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100374 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200375 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800376 ASSERT_TRUE(event.Wait(5000));
377 }
378
Henrik Boström91aa7322020-04-28 12:24:33 +0200379 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200380 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200381 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200382 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200383 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200384 event.Set();
385 });
386 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200387 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200388 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200389
Henrik Boström91aa7322020-04-28 12:24:33 +0200390 void TriggerCpuUnderuse() {
391 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200392 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200393 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200394 event.Set();
395 });
396 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200397 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200398 }
kthelgason876222f2016-11-29 01:44:11 -0800399
Henrik Boström91aa7322020-04-28 12:24:33 +0200400 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200401 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200402 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200403 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200404 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200405 event.Set();
406 });
407 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200408 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200409 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 }
419
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100421 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200422 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
423 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200424 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700425};
426
Noah Richards51db4212019-06-12 06:59:12 -0700427// Simulates simulcast behavior and makes highest stream resolutions divisible
428// by 4.
429class CroppingVideoStreamFactory
430 : public VideoEncoderConfig::VideoStreamFactoryInterface {
431 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200432 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700433
434 private:
435 std::vector<VideoStream> CreateEncoderStreams(
436 int width,
437 int height,
438 const VideoEncoderConfig& encoder_config) override {
439 std::vector<VideoStream> streams = test::CreateVideoStreams(
440 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700441 return streams;
442 }
Noah Richards51db4212019-06-12 06:59:12 -0700443};
444
sprangb1ca0732017-02-01 08:38:12 -0800445class AdaptingFrameForwarder : public test::FrameForwarder {
446 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447 explicit AdaptingFrameForwarder(TimeController* time_controller)
448 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700449 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800450
451 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200452 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800453 adaptation_enabled_ = enabled;
454 }
455
asaperssonfab67072017-04-04 05:51:49 -0700456 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200457 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800458 return adaptation_enabled_;
459 }
460
asapersson09f05612017-05-15 23:40:18 -0700461 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200462 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700463 return last_wants_;
464 }
465
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200466 absl::optional<int> last_sent_width() const { return last_width_; }
467 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800468
sprangb1ca0732017-02-01 08:38:12 -0800469 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200470 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
471 time_controller_->AdvanceTime(TimeDelta::Millis(0));
472
sprangb1ca0732017-02-01 08:38:12 -0800473 int cropped_width = 0;
474 int cropped_height = 0;
475 int out_width = 0;
476 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700477 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200478 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
479 << "w=" << video_frame.width()
480 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700481 if (adapter_.AdaptFrameResolution(
482 video_frame.width(), video_frame.height(),
483 video_frame.timestamp_us() * 1000, &cropped_width,
484 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100485 VideoFrame adapted_frame =
486 VideoFrame::Builder()
487 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
488 nullptr, out_width, out_height))
489 .set_timestamp_rtp(99)
490 .set_timestamp_ms(99)
491 .set_rotation(kVideoRotation_0)
492 .build();
sprangc5d62e22017-04-02 23:53:04 -0700493 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100494 if (video_frame.has_update_rect()) {
495 adapted_frame.set_update_rect(
496 video_frame.update_rect().ScaleWithFrame(
497 video_frame.width(), video_frame.height(), 0, 0,
498 video_frame.width(), video_frame.height(), out_width,
499 out_height));
500 }
sprangc5d62e22017-04-02 23:53:04 -0700501 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800502 last_width_.emplace(adapted_frame.width());
503 last_height_.emplace(adapted_frame.height());
504 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200505 last_width_ = absl::nullopt;
506 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700507 }
sprangb1ca0732017-02-01 08:38:12 -0800508 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200509 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800510 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800511 last_width_.emplace(video_frame.width());
512 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800513 }
514 }
515
516 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
517 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200518 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200519 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100520 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200521 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800522 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200523
524 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800525 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200526 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
527 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200528 absl::optional<int> last_width_;
529 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800530};
sprangc5d62e22017-04-02 23:53:04 -0700531
Niels Möller213618e2018-07-24 09:29:58 +0200532// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700533class MockableSendStatisticsProxy : public SendStatisticsProxy {
534 public:
535 MockableSendStatisticsProxy(Clock* clock,
536 const VideoSendStream::Config& config,
537 VideoEncoderConfig::ContentType content_type)
538 : SendStatisticsProxy(clock, config, content_type) {}
539
540 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200541 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700542 if (mock_stats_)
543 return *mock_stats_;
544 return SendStatisticsProxy::GetStats();
545 }
546
Niels Möller213618e2018-07-24 09:29:58 +0200547 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200548 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200549 if (mock_stats_)
550 return mock_stats_->input_frame_rate;
551 return SendStatisticsProxy::GetInputFrameRate();
552 }
sprangc5d62e22017-04-02 23:53:04 -0700553 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200554 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700555 mock_stats_.emplace(stats);
556 }
557
558 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200559 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700560 mock_stats_.reset();
561 }
562
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200563 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
564 on_frame_dropped_ = std::move(callback);
565 }
566
sprangc5d62e22017-04-02 23:53:04 -0700567 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200568 void OnFrameDropped(DropReason reason) override {
569 SendStatisticsProxy::OnFrameDropped(reason);
570 if (on_frame_dropped_)
571 on_frame_dropped_(reason);
572 }
573
Markus Handella3765182020-07-08 13:13:32 +0200574 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200575 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200576 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700577};
578
philipel9b058032020-02-10 11:30:00 +0100579class MockEncoderSelector
580 : public VideoEncoderFactory::EncoderSelectorInterface {
581 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200582 MOCK_METHOD(void,
583 OnCurrentEncoder,
584 (const SdpVideoFormat& format),
585 (override));
586 MOCK_METHOD(absl::optional<SdpVideoFormat>,
587 OnAvailableBitrate,
588 (const DataRate& rate),
589 (override));
590 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100591};
592
perkj803d97f2016-11-01 11:45:46 -0700593} // namespace
594
mflodmancc3d4422017-08-03 08:27:51 -0700595class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700596 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200597 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700598
mflodmancc3d4422017-08-03 08:27:51 -0700599 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700600 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700601 codec_width_(320),
602 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200603 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200604 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200605 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700606 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700608 video_send_config_,
609 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700611
612 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700613 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700614 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200615 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800616 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200617 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200618 video_send_config_.rtp.payload_name = "FAKE";
619 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700620
Per512ecb32016-09-23 15:52:06 +0200621 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200622 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200623 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
624 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
625 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100626 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700627
Niels Möllerf1338562018-04-26 09:51:47 +0200628 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800629 }
630
Niels Möllerf1338562018-04-26 09:51:47 +0200631 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700632 if (video_stream_encoder_)
633 video_stream_encoder_->Stop();
634 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200635 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
636 video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700637 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
638 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700639 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700640 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
641 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200642 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700643 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800644 }
645
Per Kjellanderdcef6412020-10-07 15:09:05 +0200646 void ResetEncoder(
647 const std::string& payload_name,
648 size_t num_streams,
649 size_t num_temporal_layers,
650 unsigned char num_spatial_layers,
651 bool screenshare,
652 VideoStreamEncoderSettings::BitrateAllocationCallbackType
653 allocation_cb_type =
654 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
655 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200656 video_send_config_.rtp.payload_name = payload_name;
Per Kjellanderdcef6412020-10-07 15:09:05 +0200657 video_send_config_.encoder_settings.allocation_cb_type = allocation_cb_type;
asapersson5f7226f2016-11-25 04:37:00 -0800658
659 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200660 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
661 num_streams, &video_encoder_config);
662 for (auto& layer : video_encoder_config.simulcast_layers) {
663 layer.num_temporal_layers = num_temporal_layers;
664 layer.max_framerate = kDefaultFramerate;
665 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100666 video_encoder_config.max_bitrate_bps =
667 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700668 video_encoder_config.content_type =
669 screenshare ? VideoEncoderConfig::ContentType::kScreen
670 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700671 if (payload_name == "VP9") {
672 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
673 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200674 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700675 video_encoder_config.encoder_specific_settings =
676 new rtc::RefCountedObject<
677 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
678 }
Niels Möllerf1338562018-04-26 09:51:47 +0200679 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700680 }
681
sprang57c2fff2017-01-16 06:24:02 -0800682 VideoFrame CreateFrame(int64_t ntp_time_ms,
683 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100684 VideoFrame frame =
685 VideoFrame::Builder()
686 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
687 destruction_event, codec_width_, codec_height_))
688 .set_timestamp_rtp(99)
689 .set_timestamp_ms(99)
690 .set_rotation(kVideoRotation_0)
691 .build();
sprang57c2fff2017-01-16 06:24:02 -0800692 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700693 return frame;
694 }
695
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100696 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
697 rtc::Event* destruction_event,
698 int offset_x) const {
699 VideoFrame frame =
700 VideoFrame::Builder()
701 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
702 destruction_event, codec_width_, codec_height_))
703 .set_timestamp_rtp(99)
704 .set_timestamp_ms(99)
705 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100706 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100707 .build();
708 frame.set_ntp_time_ms(ntp_time_ms);
709 return frame;
710 }
711
sprang57c2fff2017-01-16 06:24:02 -0800712 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100713 VideoFrame frame =
714 VideoFrame::Builder()
715 .set_video_frame_buffer(
716 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
717 .set_timestamp_rtp(99)
718 .set_timestamp_ms(99)
719 .set_rotation(kVideoRotation_0)
720 .build();
sprang57c2fff2017-01-16 06:24:02 -0800721 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700722 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700723 return frame;
724 }
725
Evan Shrubsole895556e2020-10-05 09:15:13 +0200726 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
727 VideoFrame frame =
728 VideoFrame::Builder()
729 .set_video_frame_buffer(NV12Buffer::Create(width, height))
730 .set_timestamp_rtp(99)
731 .set_timestamp_ms(99)
732 .set_rotation(kVideoRotation_0)
733 .build();
734 frame.set_ntp_time_ms(ntp_time_ms);
735 frame.set_timestamp_us(ntp_time_ms * 1000);
736 return frame;
737 }
738
Noah Richards51db4212019-06-12 06:59:12 -0700739 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
740 rtc::Event* destruction_event,
741 int width,
742 int height) const {
743 VideoFrame frame =
744 VideoFrame::Builder()
745 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
746 destruction_event, width, height))
747 .set_timestamp_rtp(99)
748 .set_timestamp_ms(99)
749 .set_rotation(kVideoRotation_0)
750 .build();
751 frame.set_ntp_time_ms(ntp_time_ms);
752 return frame;
753 }
754
Evan Shrubsole895556e2020-10-05 09:15:13 +0200755 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
756 rtc::Event* destruction_event,
757 int width,
758 int height) const {
759 VideoFrame frame = VideoFrame::Builder()
760 .set_video_frame_buffer(
761 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
762 destruction_event, width, height))
763 .set_timestamp_rtp(99)
764 .set_timestamp_ms(99)
765 .set_rotation(kVideoRotation_0)
766 .build();
767 frame.set_ntp_time_ms(ntp_time_ms);
768 return frame;
769 }
770
Noah Richards51db4212019-06-12 06:59:12 -0700771 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
772 rtc::Event* destruction_event) const {
773 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
774 codec_height_);
775 }
776
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100777 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100779 DataRate::BitsPerSec(kTargetBitrateBps),
780 DataRate::BitsPerSec(kTargetBitrateBps),
781 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100782
783 video_source_.IncomingCapturedFrame(
784 CreateFrame(1, codec_width_, codec_height_));
785 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200786 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100787 }
788
sprang4847ae62017-06-27 07:06:52 -0700789 void WaitForEncodedFrame(int64_t expected_ntp_time) {
790 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200791 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700792 }
793
794 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
795 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200796 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700797 return ok;
798 }
799
800 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
801 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200802 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700803 }
804
805 void ExpectDroppedFrame() {
806 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200807 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700808 }
809
810 bool WaitForFrame(int64_t timeout_ms) {
811 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200812 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700813 return ok;
814 }
815
perkj26091b12016-09-01 01:17:40 -0700816 class TestEncoder : public test::FakeEncoder {
817 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200818 explicit TestEncoder(TimeController* time_controller)
819 : FakeEncoder(time_controller->GetClock()),
820 time_controller_(time_controller) {
821 RTC_DCHECK(time_controller_);
822 }
perkj26091b12016-09-01 01:17:40 -0700823
asaperssonfab67072017-04-04 05:51:49 -0700824 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200825 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700826 return config_;
827 }
828
829 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200830 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700831 block_next_encode_ = true;
832 }
833
Erik Språngaed30702018-11-05 12:57:17 +0100834 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200835 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200836 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100837 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100838 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100839 info.scaling_settings = VideoEncoder::ScalingSettings(
840 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100841 }
842 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100843 for (int i = 0; i < kMaxSpatialLayers; ++i) {
844 if (temporal_layers_supported_[i]) {
845 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
846 info.fps_allocation[i].resize(num_layers);
847 }
848 }
Erik Språngaed30702018-11-05 12:57:17 +0100849 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200850
851 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100852 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200853 info.apply_alignment_to_all_simulcast_layers =
854 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200855 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100856 return info;
kthelgason876222f2016-11-29 01:44:11 -0800857 }
858
Erik Språngb7cb7b52019-02-26 15:52:33 +0100859 int32_t RegisterEncodeCompleteCallback(
860 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200861 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100862 encoded_image_callback_ = callback;
863 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
864 }
865
perkjfa10b552016-10-02 23:45:26 -0700866 void ContinueEncode() { continue_encode_event_.Set(); }
867
868 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
869 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200870 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700871 EXPECT_EQ(timestamp_, timestamp);
872 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
873 }
874
kthelgason2fc52542017-03-03 00:24:41 -0800875 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200876 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800877 quality_scaling_ = b;
878 }
kthelgasonad9010c2017-02-14 00:46:51 -0800879
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100880 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200881 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100882 requested_resolution_alignment_ = requested_resolution_alignment;
883 }
884
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200885 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
886 MutexLock lock(&local_mutex_);
887 apply_alignment_to_all_simulcast_layers_ = b;
888 }
889
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100890 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200891 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100892 is_hardware_accelerated_ = is_hardware_accelerated;
893 }
894
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100895 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
896 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200897 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100898 temporal_layers_supported_[spatial_idx] = supported;
899 }
900
Sergey Silkin6456e352019-07-08 17:56:40 +0200901 void SetResolutionBitrateLimits(
902 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200903 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200904 resolution_bitrate_limits_ = thresholds;
905 }
906
sprangfe627f32017-03-29 08:24:59 -0700907 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200908 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700909 force_init_encode_failed_ = force_failure;
910 }
911
Niels Möller6bb5ab92019-01-11 11:11:10 +0100912 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200913 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100914 rate_factor_ = rate_factor;
915 }
916
Erik Språngd7329ca2019-02-21 21:19:53 +0100917 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200918 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100919 return last_framerate_;
920 }
921
Erik Språngd7329ca2019-02-21 21:19:53 +0100922 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200923 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100924 return last_update_rect_;
925 }
926
Niels Möller87e2d782019-03-07 10:18:23 +0100927 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200928 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100929 return last_frame_types_;
930 }
931
932 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100933 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100934 keyframe ? VideoFrameType::kVideoFrameKey
935 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 {
Markus Handella3765182020-07-08 13:13:32 +0200937 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100938 last_frame_types_ = frame_type;
939 }
Niels Möllerb859b322019-03-07 12:40:01 +0100940 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100941 }
942
Erik Språngb7cb7b52019-02-26 15:52:33 +0100943 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200944 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200945 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100946 }
947
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200948 void SetEncodedImageData(
949 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200950 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200951 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200952 }
953
Erik Språngd7329ca2019-02-21 21:19:53 +0100954 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200955 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100956 expect_null_frame_ = true;
957 }
958
Erik Språng5056af02019-09-02 15:53:11 +0200959 absl::optional<VideoEncoder::RateControlParameters>
960 GetAndResetLastRateControlSettings() {
961 auto settings = last_rate_control_settings_;
962 last_rate_control_settings_.reset();
963 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 }
965
Evan Shrubsole895556e2020-10-05 09:15:13 +0200966 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
967 MutexLock lock(&local_mutex_);
968 return last_input_pixel_format_;
969 }
970
Sergey Silkin5ee69672019-07-02 14:18:34 +0200971 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200972 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200973 return num_encoder_initializations_;
974 }
975
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200976 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200977 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200978 return num_set_rates_;
979 }
980
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200981 VideoCodec video_codec() const {
982 MutexLock lock(&local_mutex_);
983 return video_codec_;
984 }
985
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200986 void SetPreferredPixelFormats(
987 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
988 pixel_formats) {
989 MutexLock lock(&local_mutex_);
990 preferred_pixel_formats_ = std::move(pixel_formats);
991 }
992
perkjfa10b552016-10-02 23:45:26 -0700993 private:
perkj26091b12016-09-01 01:17:40 -0700994 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100995 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700996 bool block_encode;
997 {
Markus Handella3765182020-07-08 13:13:32 +0200998 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100999 if (expect_null_frame_) {
1000 EXPECT_EQ(input_image.timestamp(), 0u);
1001 EXPECT_EQ(input_image.width(), 1);
1002 last_frame_types_ = *frame_types;
1003 expect_null_frame_ = false;
1004 } else {
1005 EXPECT_GT(input_image.timestamp(), timestamp_);
1006 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1007 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1008 }
perkj26091b12016-09-01 01:17:40 -07001009
1010 timestamp_ = input_image.timestamp();
1011 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001012 last_input_width_ = input_image.width();
1013 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001014 block_encode = block_next_encode_;
1015 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001016 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001017 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001018 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001019 }
Niels Möllerb859b322019-03-07 12:40:01 +01001020 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001021 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001022 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001023
perkj26091b12016-09-01 01:17:40 -07001024 return result;
1025 }
1026
Niels Möller08ae7ce2020-09-23 15:58:12 +02001027 CodecSpecificInfo EncodeHook(
1028 EncodedImage& encoded_image,
1029 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001030 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001031 {
1032 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001033 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001034 }
1035 MutexLock lock(&local_mutex_);
1036 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001037 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001038 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001039 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001040 }
1041
sprangfe627f32017-03-29 08:24:59 -07001042 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001043 const Settings& settings) override {
1044 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001045
Markus Handella3765182020-07-08 13:13:32 +02001046 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001047 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001048
1049 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001050 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001051
Erik Språng82fad3d2018-03-21 09:57:23 +01001052 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001053 // Simulate setting up temporal layers, in order to validate the life
1054 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001055 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001056 frame_buffer_controller_ =
1057 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001058 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001059 if (force_init_encode_failed_) {
1060 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001061 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001062 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001063
Erik Språngb7cb7b52019-02-26 15:52:33 +01001064 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001065 return res;
1066 }
1067
Erik Språngb7cb7b52019-02-26 15:52:33 +01001068 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001069 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001070 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1071 initialized_ = EncoderState::kUninitialized;
1072 return FakeEncoder::Release();
1073 }
1074
Erik Språng16cb8f52019-04-12 13:59:09 +02001075 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001076 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001077 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001078 VideoBitrateAllocation adjusted_rate_allocation;
1079 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1080 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001081 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001082 adjusted_rate_allocation.SetBitrate(
1083 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001084 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001085 rate_factor_));
1086 }
1087 }
1088 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001089 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001090 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001091 RateControlParameters adjusted_paramters = parameters;
1092 adjusted_paramters.bitrate = adjusted_rate_allocation;
1093 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001094 }
1095
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001096 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001097 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001098 enum class EncoderState {
1099 kUninitialized,
1100 kInitializationFailed,
1101 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001102 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1103 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001104 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001105 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1106 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1107 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1108 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1109 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1110 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001111 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1112 false;
Markus Handella3765182020-07-08 13:13:32 +02001113 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001114 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1115 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001116 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001117 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001118 absl::optional<bool>
1119 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001120 local_mutex_);
1121 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1122 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1123 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001124 absl::optional<VideoEncoder::RateControlParameters>
1125 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001126 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1127 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001128 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001129 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001130 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1131 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001132 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001133 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001134 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001135 RTC_GUARDED_BY(local_mutex_);
1136 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001137 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001138 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1139 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001140 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1141 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001142 };
1143
mflodmancc3d4422017-08-03 08:27:51 -07001144 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001145 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001146 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1147 : time_controller_(time_controller), test_encoder_(test_encoder) {
1148 RTC_DCHECK(time_controller_);
1149 }
perkj26091b12016-09-01 01:17:40 -07001150
perkj26091b12016-09-01 01:17:40 -07001151 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001152 EXPECT_TRUE(
1153 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1154 }
1155
1156 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1157 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001158 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001159 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001160 return false;
perkj26091b12016-09-01 01:17:40 -07001161 {
Markus Handella3765182020-07-08 13:13:32 +02001162 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001163 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001164 }
1165 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001166 return true;
perkj26091b12016-09-01 01:17:40 -07001167 }
1168
sprangb1ca0732017-02-01 08:38:12 -08001169 void WaitForEncodedFrame(uint32_t expected_width,
1170 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001171 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001172 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001173 }
1174
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001175 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001176 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001177 uint32_t width = 0;
1178 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001179 {
Markus Handella3765182020-07-08 13:13:32 +02001180 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001181 width = last_width_;
1182 height = last_height_;
1183 }
1184 EXPECT_EQ(expected_height, height);
1185 EXPECT_EQ(expected_width, width);
1186 }
1187
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001188 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1189 VideoRotation rotation;
1190 {
Markus Handella3765182020-07-08 13:13:32 +02001191 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001192 rotation = last_rotation_;
1193 }
1194 EXPECT_EQ(expected_rotation, rotation);
1195 }
1196
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001197 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001198
sprangc5d62e22017-04-02 23:53:04 -07001199 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001200 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1201 bool ret = encoded_frame_event_.Wait(timeout_ms);
1202 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1203 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001204 }
1205
perkj26091b12016-09-01 01:17:40 -07001206 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001207 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001208 expect_frames_ = false;
1209 }
1210
asaperssonfab67072017-04-04 05:51:49 -07001211 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001212 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001213 return number_of_reconfigurations_;
1214 }
1215
asaperssonfab67072017-04-04 05:51:49 -07001216 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001217 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001218 return min_transmit_bitrate_bps_;
1219 }
1220
Erik Språngd7329ca2019-02-21 21:19:53 +01001221 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001222 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001223 num_expected_layers_ = num_layers;
1224 }
1225
Erik Språngb7cb7b52019-02-26 15:52:33 +01001226 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001227 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001228 return last_capture_time_ms_;
1229 }
1230
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001231 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001232 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001233 return std::move(last_encoded_image_data_);
1234 }
1235
Per Kjellanderdcef6412020-10-07 15:09:05 +02001236 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1237 MutexLock lock(&mutex_);
1238 return last_bitrate_allocation_;
1239 }
1240
1241 int number_of_bitrate_allocations() const {
1242 MutexLock lock(&mutex_);
1243 return number_of_bitrate_allocations_;
1244 }
1245
Per Kjellandera9434842020-10-15 17:53:22 +02001246 VideoLayersAllocation GetLastVideoLayersAllocation() {
1247 MutexLock lock(&mutex_);
1248 return last_layers_allocation_;
1249 }
1250
1251 int number_of_layers_allocations() const {
1252 MutexLock lock(&mutex_);
1253 return number_of_layers_allocations_;
1254 }
1255
perkj26091b12016-09-01 01:17:40 -07001256 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001257 Result OnEncodedImage(
1258 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001259 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001260 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001261 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001262 last_encoded_image_data_ = std::vector<uint8_t>(
1263 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001264 uint32_t timestamp = encoded_image.Timestamp();
1265 if (last_timestamp_ != timestamp) {
1266 num_received_layers_ = 1;
1267 } else {
1268 ++num_received_layers_;
1269 }
1270 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001271 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001272 last_width_ = encoded_image._encodedWidth;
1273 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001274 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001275 if (num_received_layers_ == num_expected_layers_) {
1276 encoded_frame_event_.Set();
1277 }
sprangb1ca0732017-02-01 08:38:12 -08001278 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001279 }
1280
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001281 void OnEncoderConfigurationChanged(
1282 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001283 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001284 VideoEncoderConfig::ContentType content_type,
1285 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001286 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001287 ++number_of_reconfigurations_;
1288 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1289 }
1290
Per Kjellanderdcef6412020-10-07 15:09:05 +02001291 void OnBitrateAllocationUpdated(
1292 const VideoBitrateAllocation& allocation) override {
1293 MutexLock lock(&mutex_);
1294 ++number_of_bitrate_allocations_;
1295 last_bitrate_allocation_ = allocation;
1296 }
1297
Per Kjellandera9434842020-10-15 17:53:22 +02001298 void OnVideoLayersAllocationUpdated(
1299 VideoLayersAllocation allocation) override {
1300 MutexLock lock(&mutex_);
1301 ++number_of_layers_allocations_;
1302 last_layers_allocation_ = allocation;
1303 rtc::StringBuilder log;
1304 for (const auto& layer : allocation.active_spatial_layers) {
1305 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1306 << "[";
1307 for (const auto target_bitrate :
1308 layer.target_bitrate_per_temporal_layer) {
1309 log << target_bitrate.kbps() << ",";
1310 }
1311 log << "]";
1312 }
1313 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1314 }
1315
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001316 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001317 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001318 TestEncoder* test_encoder_;
1319 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001320 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001321 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001322 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001323 uint32_t last_height_ = 0;
1324 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001325 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001326 size_t num_expected_layers_ = 1;
1327 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001328 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001329 int number_of_reconfigurations_ = 0;
1330 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001331 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1332 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001333 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1334 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001335 };
1336
Sergey Silkin5ee69672019-07-02 14:18:34 +02001337 class VideoBitrateAllocatorProxyFactory
1338 : public VideoBitrateAllocatorFactory {
1339 public:
1340 VideoBitrateAllocatorProxyFactory()
1341 : bitrate_allocator_factory_(
1342 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1343
1344 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1345 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001346 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001347 codec_config_ = codec;
1348 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1349 }
1350
1351 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001352 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001353 return codec_config_;
1354 }
1355
1356 private:
1357 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1358
Markus Handella3765182020-07-08 13:13:32 +02001359 mutable Mutex mutex_;
1360 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001361 };
1362
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001363 Clock* clock() { return time_controller_.GetClock(); }
1364 void AdvanceTime(TimeDelta duration) {
1365 time_controller_.AdvanceTime(duration);
1366 }
1367
1368 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1369
1370 protected:
1371 virtual TaskQueueFactory* GetTaskQueueFactory() {
1372 return time_controller_.GetTaskQueueFactory();
1373 }
1374
1375 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001376 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001377 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001378 int codec_width_;
1379 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001380 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001381 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001382 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001383 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001384 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001385 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001386 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001387 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001388};
1389
mflodmancc3d4422017-08-03 08:27:51 -07001390TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001391 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001392 DataRate::BitsPerSec(kTargetBitrateBps),
1393 DataRate::BitsPerSec(kTargetBitrateBps),
1394 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001395 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001396 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001397 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001398 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001399 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001400}
1401
mflodmancc3d4422017-08-03 08:27:51 -07001402TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001403 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001404 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001405 // The encoder will cache up to one frame for a short duration. Adding two
1406 // frames means that the first frame will be dropped and the second frame will
1407 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001408 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001409 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001410 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001411 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001412
Henrik Boström381d1092020-05-12 18:49:07 +02001413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001414 DataRate::BitsPerSec(kTargetBitrateBps),
1415 DataRate::BitsPerSec(kTargetBitrateBps),
1416 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001417
Sebastian Janssona3177052018-04-10 13:05:49 +02001418 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001419 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001420 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1421
1422 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001423 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001424}
1425
mflodmancc3d4422017-08-03 08:27:51 -07001426TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001428 DataRate::BitsPerSec(kTargetBitrateBps),
1429 DataRate::BitsPerSec(kTargetBitrateBps),
1430 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001431 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001432 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001433
Henrik Boström381d1092020-05-12 18:49:07 +02001434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1435 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1436 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001437 // The encoder will cache up to one frame for a short duration. Adding two
1438 // frames means that the first frame will be dropped and the second frame will
1439 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001440 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001441 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001442
Henrik Boström381d1092020-05-12 18:49:07 +02001443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001444 DataRate::BitsPerSec(kTargetBitrateBps),
1445 DataRate::BitsPerSec(kTargetBitrateBps),
1446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001447 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001448 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1449 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001450 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001451}
1452
mflodmancc3d4422017-08-03 08:27:51 -07001453TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001455 DataRate::BitsPerSec(kTargetBitrateBps),
1456 DataRate::BitsPerSec(kTargetBitrateBps),
1457 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001458 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001459 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001460
1461 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001462 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001463
perkja49cbd32016-09-16 07:53:41 -07001464 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001465 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001466 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001467}
1468
mflodmancc3d4422017-08-03 08:27:51 -07001469TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001470 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001471 DataRate::BitsPerSec(kTargetBitrateBps),
1472 DataRate::BitsPerSec(kTargetBitrateBps),
1473 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001474
perkja49cbd32016-09-16 07:53:41 -07001475 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001476 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001477
mflodmancc3d4422017-08-03 08:27:51 -07001478 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001479 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001480 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001481 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1482 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001483}
1484
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001485class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1486 public:
1487 VideoStreamEncoderBlockedTest() {}
1488
1489 TaskQueueFactory* GetTaskQueueFactory() override {
1490 return task_queue_factory_.get();
1491 }
1492
1493 private:
1494 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1495 CreateDefaultTaskQueueFactory();
1496};
1497
1498TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001500 DataRate::BitsPerSec(kTargetBitrateBps),
1501 DataRate::BitsPerSec(kTargetBitrateBps),
1502 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001503
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001504 int dropped_count = 0;
1505 stats_proxy_->SetDroppedFrameCallback(
1506 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1507 ++dropped_count;
1508 });
1509
perkj26091b12016-09-01 01:17:40 -07001510 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001511 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001512 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001513 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1514 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001515 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1516 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001517 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001518 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001519
mflodmancc3d4422017-08-03 08:27:51 -07001520 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001521
1522 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001523}
1524
Noah Richards51db4212019-06-12 06:59:12 -07001525TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001527 DataRate::BitsPerSec(kTargetBitrateBps),
1528 DataRate::BitsPerSec(kTargetBitrateBps),
1529 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001530
1531 rtc::Event frame_destroyed_event;
1532 video_source_.IncomingCapturedFrame(
1533 CreateFakeNativeFrame(1, &frame_destroyed_event));
1534 ExpectDroppedFrame();
1535 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1536 video_stream_encoder_->Stop();
1537}
1538
1539TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1540 // Use the cropping factory.
1541 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001542 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001543 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1544 kMaxPayloadLength);
1545 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1546
1547 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001549 DataRate::BitsPerSec(kTargetBitrateBps),
1550 DataRate::BitsPerSec(kTargetBitrateBps),
1551 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001552 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1553 WaitForEncodedFrame(1);
1554 // The encoder will have been configured once.
1555 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1556 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1557 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1558
1559 // Now send in a fake frame that needs to be cropped as the width/height
1560 // aren't divisible by 4 (see CreateEncoderStreams above).
1561 rtc::Event frame_destroyed_event;
1562 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1563 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1564 ExpectDroppedFrame();
1565 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1566 video_stream_encoder_->Stop();
1567}
1568
Evan Shrubsole895556e2020-10-05 09:15:13 +02001569TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1571 DataRate::BitsPerSec(kTargetBitrateBps),
1572 DataRate::BitsPerSec(kTargetBitrateBps),
1573 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1574
1575 video_source_.IncomingCapturedFrame(
1576 CreateNV12Frame(1, codec_width_, codec_height_));
1577 WaitForEncodedFrame(1);
1578 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1579 fake_encoder_.GetLastInputPixelFormat());
1580 video_stream_encoder_->Stop();
1581}
1582
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001583TEST_F(VideoStreamEncoderTest,
1584 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1586 DataRate::BitsPerSec(kTargetBitrateBps),
1587 DataRate::BitsPerSec(kTargetBitrateBps),
1588 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1589
1590 fake_encoder_.SetPreferredPixelFormats({});
1591
1592 rtc::Event frame_destroyed_event;
1593 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1594 1, &frame_destroyed_event, codec_width_, codec_height_));
1595 WaitForEncodedFrame(1);
1596 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1597 fake_encoder_.GetLastInputPixelFormat());
1598 video_stream_encoder_->Stop();
1599}
1600
1601TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1603 DataRate::BitsPerSec(kTargetBitrateBps),
1604 DataRate::BitsPerSec(kTargetBitrateBps),
1605 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1606
1607 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1608
1609 rtc::Event frame_destroyed_event;
1610 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1611 1, &frame_destroyed_event, codec_width_, codec_height_));
1612 WaitForEncodedFrame(1);
1613 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1614 fake_encoder_.GetLastInputPixelFormat());
1615 video_stream_encoder_->Stop();
1616}
1617
1618TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1620 DataRate::BitsPerSec(kTargetBitrateBps),
1621 DataRate::BitsPerSec(kTargetBitrateBps),
1622 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1623
1624 // Fake NV12 native frame does not allow mapping to I444.
1625 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1626
1627 rtc::Event frame_destroyed_event;
1628 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1629 1, &frame_destroyed_event, codec_width_, codec_height_));
1630 WaitForEncodedFrame(1);
1631 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1632 fake_encoder_.GetLastInputPixelFormat());
1633 video_stream_encoder_->Stop();
1634}
1635
Evan Shrubsole895556e2020-10-05 09:15:13 +02001636TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1638 DataRate::BitsPerSec(kTargetBitrateBps),
1639 DataRate::BitsPerSec(kTargetBitrateBps),
1640 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1641
1642 rtc::Event frame_destroyed_event;
1643 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1644 1, &frame_destroyed_event, codec_width_, codec_height_));
1645 WaitForEncodedFrame(1);
1646 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1647 fake_encoder_.GetLastInputPixelFormat());
1648 video_stream_encoder_->Stop();
1649}
1650
Ying Wang9b881ab2020-02-07 14:29:32 +01001651TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001653 DataRate::BitsPerSec(kTargetBitrateBps),
1654 DataRate::BitsPerSec(kTargetBitrateBps),
1655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001656 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1657 WaitForEncodedFrame(1);
1658
Henrik Boström381d1092020-05-12 18:49:07 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001660 DataRate::BitsPerSec(kTargetBitrateBps),
1661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001663 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1664 // frames. Adding two frames means that the first frame will be dropped and
1665 // the second frame will be sent to the encoder.
1666 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1667 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1668 WaitForEncodedFrame(3);
1669 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1670 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1671 WaitForEncodedFrame(5);
1672 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1673 video_stream_encoder_->Stop();
1674}
1675
mflodmancc3d4422017-08-03 08:27:51 -07001676TEST_F(VideoStreamEncoderTest,
1677 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001679 DataRate::BitsPerSec(kTargetBitrateBps),
1680 DataRate::BitsPerSec(kTargetBitrateBps),
1681 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001682 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001683
1684 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001685 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001686 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001687 // The encoder will have been configured once when the first frame is
1688 // received.
1689 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001690
1691 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001692 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001693 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001694 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001695 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001696
1697 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001698 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001699 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001700 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001701 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001702
mflodmancc3d4422017-08-03 08:27:51 -07001703 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001704}
1705
mflodmancc3d4422017-08-03 08:27:51 -07001706TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001708 DataRate::BitsPerSec(kTargetBitrateBps),
1709 DataRate::BitsPerSec(kTargetBitrateBps),
1710 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001711
1712 // Capture a frame and wait for it to synchronize with the encoder thread.
1713 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001714 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001715 // The encoder will have been configured once.
1716 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001717 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1718 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1719
1720 codec_width_ *= 2;
1721 codec_height_ *= 2;
1722 // Capture a frame with a higher resolution and wait for it to synchronize
1723 // with the encoder thread.
1724 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001725 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001726 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1727 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001728 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001729
mflodmancc3d4422017-08-03 08:27:51 -07001730 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001731}
1732
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001733TEST_F(VideoStreamEncoderTest,
1734 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001736 DataRate::BitsPerSec(kTargetBitrateBps),
1737 DataRate::BitsPerSec(kTargetBitrateBps),
1738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001739
1740 // Capture a frame and wait for it to synchronize with the encoder thread.
1741 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1742 WaitForEncodedFrame(1);
1743
1744 VideoEncoderConfig video_encoder_config;
1745 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1746 // Changing the max payload data length recreates encoder.
1747 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1748 kMaxPayloadLength / 2);
1749
1750 // Capture a frame and wait for it to synchronize with the encoder thread.
1751 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1752 WaitForEncodedFrame(2);
1753 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1754
1755 video_stream_encoder_->Stop();
1756}
1757
Sergey Silkin5ee69672019-07-02 14:18:34 +02001758TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001759 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001760 DataRate::BitsPerSec(kTargetBitrateBps),
1761 DataRate::BitsPerSec(kTargetBitrateBps),
1762 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001763
1764 VideoEncoderConfig video_encoder_config;
1765 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1766 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1767 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1768 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1769 kMaxPayloadLength);
1770
1771 // Capture a frame and wait for it to synchronize with the encoder thread.
1772 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1773 WaitForEncodedFrame(1);
1774 // The encoder will have been configured once when the first frame is
1775 // received.
1776 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1777 EXPECT_EQ(kTargetBitrateBps,
1778 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1779 EXPECT_EQ(kStartBitrateBps,
1780 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1781
Sergey Silkin6456e352019-07-08 17:56:40 +02001782 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1783 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001784 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1785 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1786 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1787 kMaxPayloadLength);
1788
1789 // Capture a frame and wait for it to synchronize with the encoder thread.
1790 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1791 WaitForEncodedFrame(2);
1792 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1793 // Bitrate limits have changed - rate allocator should be reconfigured,
1794 // encoder should not be reconfigured.
1795 EXPECT_EQ(kTargetBitrateBps * 2,
1796 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1797 EXPECT_EQ(kStartBitrateBps * 2,
1798 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1799 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1800
1801 video_stream_encoder_->Stop();
1802}
1803
Sergey Silkin6456e352019-07-08 17:56:40 +02001804TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001805 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001807 DataRate::BitsPerSec(kTargetBitrateBps),
1808 DataRate::BitsPerSec(kTargetBitrateBps),
1809 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001810
Sergey Silkincd02eba2020-01-20 14:48:40 +01001811 const uint32_t kMinEncBitrateKbps = 100;
1812 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001813 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001814 /*frame_size_pixels=*/codec_width_ * codec_height_,
1815 /*min_start_bitrate_bps=*/0,
1816 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1817 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001818 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1819
Sergey Silkincd02eba2020-01-20 14:48:40 +01001820 VideoEncoderConfig video_encoder_config;
1821 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1822 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1823 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1824 (kMinEncBitrateKbps + 1) * 1000;
1825 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1826 kMaxPayloadLength);
1827
1828 // When both encoder and app provide bitrate limits, the intersection of
1829 // provided sets should be used.
1830 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1831 WaitForEncodedFrame(1);
1832 EXPECT_EQ(kMaxEncBitrateKbps,
1833 bitrate_allocator_factory_.codec_config().maxBitrate);
1834 EXPECT_EQ(kMinEncBitrateKbps + 1,
1835 bitrate_allocator_factory_.codec_config().minBitrate);
1836
1837 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1838 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1839 (kMinEncBitrateKbps - 1) * 1000;
1840 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1841 kMaxPayloadLength);
1842 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001843 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001844 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001845 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001846 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001847 bitrate_allocator_factory_.codec_config().minBitrate);
1848
Sergey Silkincd02eba2020-01-20 14:48:40 +01001849 video_stream_encoder_->Stop();
1850}
1851
1852TEST_F(VideoStreamEncoderTest,
1853 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001855 DataRate::BitsPerSec(kTargetBitrateBps),
1856 DataRate::BitsPerSec(kTargetBitrateBps),
1857 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001858
1859 const uint32_t kMinAppBitrateKbps = 100;
1860 const uint32_t kMaxAppBitrateKbps = 200;
1861 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1862 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1863 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1864 /*frame_size_pixels=*/codec_width_ * codec_height_,
1865 /*min_start_bitrate_bps=*/0,
1866 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1867 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1868 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1869
1870 VideoEncoderConfig video_encoder_config;
1871 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1872 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1873 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1874 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001875 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1876 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001877
Sergey Silkincd02eba2020-01-20 14:48:40 +01001878 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1879 WaitForEncodedFrame(1);
1880 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001881 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001882 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001884
1885 video_stream_encoder_->Stop();
1886}
1887
1888TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001889 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001891 DataRate::BitsPerSec(kTargetBitrateBps),
1892 DataRate::BitsPerSec(kTargetBitrateBps),
1893 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001894
1895 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001896 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001897 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001898 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001899 fake_encoder_.SetResolutionBitrateLimits(
1900 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1901
1902 VideoEncoderConfig video_encoder_config;
1903 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1904 video_encoder_config.max_bitrate_bps = 0;
1905 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1906 kMaxPayloadLength);
1907
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001908 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001909 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1910 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001911 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1912 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001913 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1914 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1915
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001916 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001917 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1918 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001919 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1920 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001921 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1922 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1923
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001924 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001925 // encoder for 360p should be used.
1926 video_source_.IncomingCapturedFrame(
1927 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1928 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001929 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1930 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001931 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1932 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1933
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001934 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001935 // ignored.
1936 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1937 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001938 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1939 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001940 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1941 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1943 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001944 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1945 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1946
1947 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1948 // for 270p should be used.
1949 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1950 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001951 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1952 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001953 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1954 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1955
1956 video_stream_encoder_->Stop();
1957}
1958
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001959TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001961 DataRate::BitsPerSec(kTargetBitrateBps),
1962 DataRate::BitsPerSec(kTargetBitrateBps),
1963 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001964
1965 VideoEncoderConfig video_encoder_config;
1966 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1967 video_encoder_config.max_bitrate_bps = 0;
1968 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1969 kMaxPayloadLength);
1970
1971 // Encode 720p frame to get the default encoder target bitrate.
1972 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1973 WaitForEncodedFrame(1);
1974 const uint32_t kDefaultTargetBitrateFor720pKbps =
1975 bitrate_allocator_factory_.codec_config()
1976 .simulcastStream[0]
1977 .targetBitrate;
1978
1979 // Set the max recommended encoder bitrate to something lower than the default
1980 // target bitrate.
1981 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1982 1280 * 720, 10 * 1000, 10 * 1000,
1983 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1984 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1985
1986 // Change resolution to trigger encoder reinitialization.
1987 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1988 WaitForEncodedFrame(2);
1989 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1990 WaitForEncodedFrame(3);
1991
1992 // Ensure the target bitrate is capped by the max bitrate.
1993 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1994 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1995 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1996 .simulcastStream[0]
1997 .targetBitrate *
1998 1000,
1999 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2000
2001 video_stream_encoder_->Stop();
2002}
2003
mflodmancc3d4422017-08-03 08:27:51 -07002004TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002005 EXPECT_TRUE(video_source_.has_sinks());
2006 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002007 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002008 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002009 EXPECT_FALSE(video_source_.has_sinks());
2010 EXPECT_TRUE(new_video_source.has_sinks());
2011
mflodmancc3d4422017-08-03 08:27:51 -07002012 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002013}
2014
mflodmancc3d4422017-08-03 08:27:51 -07002015TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002016 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002017 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002018 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002019 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002020}
2021
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002022class ResolutionAlignmentTest
2023 : public VideoStreamEncoderTest,
2024 public ::testing::WithParamInterface<
2025 ::testing::tuple<int, std::vector<double>>> {
2026 public:
2027 ResolutionAlignmentTest()
2028 : requested_alignment_(::testing::get<0>(GetParam())),
2029 scale_factors_(::testing::get<1>(GetParam())) {}
2030
2031 protected:
2032 const int requested_alignment_;
2033 const std::vector<double> scale_factors_;
2034};
2035
2036INSTANTIATE_TEST_SUITE_P(
2037 AlignmentAndScaleFactors,
2038 ResolutionAlignmentTest,
2039 ::testing::Combine(
2040 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2041 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2042 std::vector<double>{-1.0, -1.0},
2043 std::vector<double>{-1.0, -1.0, -1.0},
2044 std::vector<double>{4.0, 2.0, 1.0},
2045 std::vector<double>{9999.0, -1.0, 1.0},
2046 std::vector<double>{3.99, 2.01, 1.0},
2047 std::vector<double>{4.9, 1.7, 1.25},
2048 std::vector<double>{10.0, 4.0, 3.0},
2049 std::vector<double>{1.75, 3.5},
2050 std::vector<double>{1.5, 2.5},
2051 std::vector<double>{1.3, 1.0})));
2052
2053TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2054 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002055 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002056 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2057 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2058
2059 // Fill config with the scaling factor by which to reduce encoding size.
2060 const int num_streams = scale_factors_.size();
2061 VideoEncoderConfig config;
2062 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2063 for (int i = 0; i < num_streams; ++i) {
2064 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2065 }
2066 config.video_stream_factory =
2067 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2068 "VP8", /*max qp*/ 56, /*screencast*/ false,
2069 /*screenshare enabled*/ false);
2070 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2071
Henrik Boström381d1092020-05-12 18:49:07 +02002072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002073 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2074 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2075 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2076 // Wait for all layers before triggering event.
2077 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002078
2079 // On the 1st frame, we should have initialized the encoder and
2080 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002081 int64_t timestamp_ms = kFrameIntervalMs;
2082 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2083 WaitForEncodedFrame(timestamp_ms);
2084 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002085
2086 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2087 // (It's up the to the encoder to potentially drop the previous frame,
2088 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002089 timestamp_ms += kFrameIntervalMs;
2090 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2091 WaitForEncodedFrame(timestamp_ms);
2092 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2093
2094 VideoCodec codec = fake_encoder_.video_codec();
2095 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2096 // Frame size should be a multiple of the requested alignment.
2097 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2098 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2099 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2100 // Aspect ratio should match.
2101 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2102 codec.height * codec.simulcastStream[i].width);
2103 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002104
2105 video_stream_encoder_->Stop();
2106}
2107
Jonathan Yubc771b72017-12-08 17:04:29 -08002108TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2109 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002110 const int kWidth = 1280;
2111 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002112
2113 // We rely on the automatic resolution adaptation, but we handle framerate
2114 // adaptation manually by mocking the stats proxy.
2115 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002116
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002117 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002119 DataRate::BitsPerSec(kTargetBitrateBps),
2120 DataRate::BitsPerSec(kTargetBitrateBps),
2121 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002122 video_stream_encoder_->SetSource(&video_source_,
2123 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002124 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002125 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002126 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002127 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2128
Jonathan Yubc771b72017-12-08 17:04:29 -08002129 // Adapt down as far as possible.
2130 rtc::VideoSinkWants last_wants;
2131 int64_t t = 1;
2132 int loop_count = 0;
2133 do {
2134 ++loop_count;
2135 last_wants = video_source_.sink_wants();
2136
2137 // Simulate the framerate we've been asked to adapt to.
2138 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2139 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2140 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2141 mock_stats.input_frame_rate = fps;
2142 stats_proxy_->SetMockStats(mock_stats);
2143
2144 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2145 sink_.WaitForEncodedFrame(t);
2146 t += frame_interval_ms;
2147
mflodmancc3d4422017-08-03 08:27:51 -07002148 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002149 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002150 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002151 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2152 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002153 } while (video_source_.sink_wants().max_pixel_count <
2154 last_wants.max_pixel_count ||
2155 video_source_.sink_wants().max_framerate_fps <
2156 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002157
Jonathan Yubc771b72017-12-08 17:04:29 -08002158 // Verify that we've adapted all the way down.
2159 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002160 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002161 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2162 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002163 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002164 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2165 *video_source_.last_sent_height());
2166 EXPECT_EQ(kMinBalancedFramerateFps,
2167 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002168
Jonathan Yubc771b72017-12-08 17:04:29 -08002169 // Adapt back up the same number of times we adapted down.
2170 for (int i = 0; i < loop_count - 1; ++i) {
2171 last_wants = video_source_.sink_wants();
2172
2173 // Simulate the framerate we've been asked to adapt to.
2174 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2175 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2176 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2177 mock_stats.input_frame_rate = fps;
2178 stats_proxy_->SetMockStats(mock_stats);
2179
2180 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2181 sink_.WaitForEncodedFrame(t);
2182 t += frame_interval_ms;
2183
Henrik Boström91aa7322020-04-28 12:24:33 +02002184 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002185 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002186 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002187 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2188 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002189 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2190 last_wants.max_pixel_count ||
2191 video_source_.sink_wants().max_framerate_fps >
2192 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002193 }
2194
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002195 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002196 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002197 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002198 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2199 EXPECT_EQ((loop_count - 1) * 2,
2200 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002201
mflodmancc3d4422017-08-03 08:27:51 -07002202 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002203}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002204
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002205TEST_F(VideoStreamEncoderTest,
2206 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2207 video_stream_encoder_->OnBitrateUpdated(
2208 DataRate::BitsPerSec(kTargetBitrateBps),
2209 DataRate::BitsPerSec(kTargetBitrateBps),
2210 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002211 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002212
2213 const int kFrameWidth = 1280;
2214 const int kFrameHeight = 720;
2215
2216 int64_t ntp_time = kFrameIntervalMs;
2217
2218 // Force an input frame rate to be available, or the adaptation call won't
2219 // know what framerate to adapt form.
2220 const int kInputFps = 30;
2221 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2222 stats.input_frame_rate = kInputFps;
2223 stats_proxy_->SetMockStats(stats);
2224
2225 video_source_.set_adaptation_enabled(true);
2226 video_stream_encoder_->SetSource(
2227 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002228 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002229 video_source_.IncomingCapturedFrame(
2230 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2231 sink_.WaitForEncodedFrame(ntp_time);
2232 ntp_time += kFrameIntervalMs;
2233
2234 // Trigger CPU overuse.
2235 video_stream_encoder_->TriggerCpuOveruse();
2236 video_source_.IncomingCapturedFrame(
2237 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2238 sink_.WaitForEncodedFrame(ntp_time);
2239 ntp_time += kFrameIntervalMs;
2240
2241 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2242 EXPECT_EQ(std::numeric_limits<int>::max(),
2243 video_source_.sink_wants().max_pixel_count);
2244 // Some framerate constraint should be set.
2245 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2246 EXPECT_LT(restricted_fps, kInputFps);
2247 video_source_.IncomingCapturedFrame(
2248 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2249 sink_.WaitForEncodedFrame(ntp_time);
2250 ntp_time += 100;
2251
Henrik Boström2671dac2020-05-19 16:29:09 +02002252 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002253 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2254 // Give the encoder queue time to process the change in degradation preference
2255 // by waiting for an encoded frame.
2256 video_source_.IncomingCapturedFrame(
2257 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2258 sink_.WaitForEncodedFrame(ntp_time);
2259 ntp_time += kFrameIntervalMs;
2260
2261 video_stream_encoder_->TriggerQualityLow();
2262 video_source_.IncomingCapturedFrame(
2263 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2264 sink_.WaitForEncodedFrame(ntp_time);
2265 ntp_time += kFrameIntervalMs;
2266
2267 // Some resolution constraint should be set.
2268 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2269 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2270 kFrameWidth * kFrameHeight);
2271 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2272
2273 int pixel_count = video_source_.sink_wants().max_pixel_count;
2274 // Triggering a CPU underuse should not change the sink wants since it has
2275 // not been overused for resolution since we changed degradation preference.
2276 video_stream_encoder_->TriggerCpuUnderuse();
2277 video_source_.IncomingCapturedFrame(
2278 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2279 sink_.WaitForEncodedFrame(ntp_time);
2280 ntp_time += kFrameIntervalMs;
2281 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2282 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2283
Evan Shrubsole64469032020-06-11 10:45:29 +02002284 // Change the degradation preference back. CPU underuse should not adapt since
2285 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002286 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002287 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2288 video_source_.IncomingCapturedFrame(
2289 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2290 sink_.WaitForEncodedFrame(ntp_time);
2291 ntp_time += 100;
2292 // Resolution adaptations is gone after changing degradation preference.
2293 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2294 EXPECT_EQ(std::numeric_limits<int>::max(),
2295 video_source_.sink_wants().max_pixel_count);
2296 // The fps adaptation from above is now back.
2297 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2298
2299 // Trigger CPU underuse.
2300 video_stream_encoder_->TriggerCpuUnderuse();
2301 video_source_.IncomingCapturedFrame(
2302 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2303 sink_.WaitForEncodedFrame(ntp_time);
2304 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002305 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2306
2307 // Trigger QP underuse, fps should return to normal.
2308 video_stream_encoder_->TriggerQualityHigh();
2309 video_source_.IncomingCapturedFrame(
2310 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2311 sink_.WaitForEncodedFrame(ntp_time);
2312 ntp_time += kFrameIntervalMs;
2313 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002314
2315 video_stream_encoder_->Stop();
2316}
2317
mflodmancc3d4422017-08-03 08:27:51 -07002318TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002320 DataRate::BitsPerSec(kTargetBitrateBps),
2321 DataRate::BitsPerSec(kTargetBitrateBps),
2322 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002323 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002324
sprangc5d62e22017-04-02 23:53:04 -07002325 const int kFrameWidth = 1280;
2326 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002327
Åsa Persson8c1bf952018-09-13 10:42:19 +02002328 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002329
kthelgason5e13d412016-12-01 03:59:51 -08002330 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002331 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002332 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002333 frame_timestamp += kFrameIntervalMs;
2334
perkj803d97f2016-11-01 11:45:46 -07002335 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002336 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002337 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002338 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002339 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002340 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002341
asapersson0944a802017-04-07 00:57:58 -07002342 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002343 // wanted resolution.
2344 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2345 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2346 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002347 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002348
2349 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002350 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002351 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002352 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002353 // Give the encoder queue time to process the change in degradation preference
2354 // by waiting for an encoded frame.
2355 new_video_source.IncomingCapturedFrame(
2356 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2357 sink_.WaitForEncodedFrame(frame_timestamp);
2358 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002359 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002360 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002361
sprangc5d62e22017-04-02 23:53:04 -07002362 // Force an input frame rate to be available, or the adaptation call won't
2363 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002364 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002365 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002366 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002367 stats_proxy_->SetMockStats(stats);
2368
mflodmancc3d4422017-08-03 08:27:51 -07002369 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002370 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002371 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002372 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002373 frame_timestamp += kFrameIntervalMs;
2374
2375 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002376 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002377 EXPECT_EQ(std::numeric_limits<int>::max(),
2378 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002379 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002380
asapersson02465b82017-04-10 01:12:52 -07002381 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002382 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2383 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002384 // Give the encoder queue time to process the change in degradation preference
2385 // by waiting for an encoded frame.
2386 new_video_source.IncomingCapturedFrame(
2387 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2388 sink_.WaitForEncodedFrame(frame_timestamp);
2389 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002390 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002391
mflodmancc3d4422017-08-03 08:27:51 -07002392 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002393 new_video_source.IncomingCapturedFrame(
2394 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002395 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002396 frame_timestamp += kFrameIntervalMs;
2397
2398 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002399 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002400
2401 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002402 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002403 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002404 // Give the encoder queue time to process the change in degradation preference
2405 // by waiting for an encoded frame.
2406 new_video_source.IncomingCapturedFrame(
2407 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2408 sink_.WaitForEncodedFrame(frame_timestamp);
2409 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002410 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2411 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002412 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002413 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002414
2415 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002416 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002417 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002418 // Give the encoder queue time to process the change in degradation preference
2419 // by waiting for an encoded frame.
2420 new_video_source.IncomingCapturedFrame(
2421 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2422 sink_.WaitForEncodedFrame(frame_timestamp);
2423 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002424 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2425 EXPECT_EQ(std::numeric_limits<int>::max(),
2426 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002427 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002428
mflodmancc3d4422017-08-03 08:27:51 -07002429 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002430}
2431
mflodmancc3d4422017-08-03 08:27:51 -07002432TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002434 DataRate::BitsPerSec(kTargetBitrateBps),
2435 DataRate::BitsPerSec(kTargetBitrateBps),
2436 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002437
asaperssonfab67072017-04-04 05:51:49 -07002438 const int kWidth = 1280;
2439 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002440 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002441 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002442 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2443 EXPECT_FALSE(stats.bw_limited_resolution);
2444 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2445
2446 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002447 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002448 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002449 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002450
2451 stats = stats_proxy_->GetStats();
2452 EXPECT_TRUE(stats.bw_limited_resolution);
2453 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2454
2455 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002456 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002457 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002458 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002459
2460 stats = stats_proxy_->GetStats();
2461 EXPECT_FALSE(stats.bw_limited_resolution);
2462 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2463 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2464
mflodmancc3d4422017-08-03 08:27:51 -07002465 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002466}
2467
mflodmancc3d4422017-08-03 08:27:51 -07002468TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002470 DataRate::BitsPerSec(kTargetBitrateBps),
2471 DataRate::BitsPerSec(kTargetBitrateBps),
2472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002473
2474 const int kWidth = 1280;
2475 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002476 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002477 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002478 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2479 EXPECT_FALSE(stats.cpu_limited_resolution);
2480 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2481
2482 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002483 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002484 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002485 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002486
2487 stats = stats_proxy_->GetStats();
2488 EXPECT_TRUE(stats.cpu_limited_resolution);
2489 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2490
2491 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002492 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002493 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002494 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002495
2496 stats = stats_proxy_->GetStats();
2497 EXPECT_FALSE(stats.cpu_limited_resolution);
2498 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002499 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002500
mflodmancc3d4422017-08-03 08:27:51 -07002501 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002502}
2503
mflodmancc3d4422017-08-03 08:27:51 -07002504TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002506 DataRate::BitsPerSec(kTargetBitrateBps),
2507 DataRate::BitsPerSec(kTargetBitrateBps),
2508 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002509
asaperssonfab67072017-04-04 05:51:49 -07002510 const int kWidth = 1280;
2511 const int kHeight = 720;
2512 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002513 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002514 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002515 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002516 EXPECT_FALSE(stats.cpu_limited_resolution);
2517 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2518
asaperssonfab67072017-04-04 05:51:49 -07002519 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002520 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002521 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002522 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002523 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002524 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002525 EXPECT_TRUE(stats.cpu_limited_resolution);
2526 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2527
2528 // Set new source with adaptation still enabled.
2529 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002530 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002531 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002532
asaperssonfab67072017-04-04 05:51:49 -07002533 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002534 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002535 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002536 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002537 EXPECT_TRUE(stats.cpu_limited_resolution);
2538 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2539
2540 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002541 video_stream_encoder_->SetSource(&new_video_source,
2542 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002543
asaperssonfab67072017-04-04 05:51:49 -07002544 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002545 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002546 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002547 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002548 EXPECT_FALSE(stats.cpu_limited_resolution);
2549 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2550
2551 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002552 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002554
asaperssonfab67072017-04-04 05:51:49 -07002555 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002556 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002557 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002558 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002559 EXPECT_TRUE(stats.cpu_limited_resolution);
2560 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2561
asaperssonfab67072017-04-04 05:51:49 -07002562 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002563 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002564 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002565 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002566 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002567 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002568 EXPECT_FALSE(stats.cpu_limited_resolution);
2569 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002570 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002571
mflodmancc3d4422017-08-03 08:27:51 -07002572 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002573}
2574
mflodmancc3d4422017-08-03 08:27:51 -07002575TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002577 DataRate::BitsPerSec(kTargetBitrateBps),
2578 DataRate::BitsPerSec(kTargetBitrateBps),
2579 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002580
asaperssonfab67072017-04-04 05:51:49 -07002581 const int kWidth = 1280;
2582 const int kHeight = 720;
2583 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002584 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002585 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002586 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002587 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002588 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002589
2590 // Set new source with adaptation still enabled.
2591 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002592 video_stream_encoder_->SetSource(&new_video_source,
2593 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002594
asaperssonfab67072017-04-04 05:51:49 -07002595 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002596 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002597 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002598 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002599 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002600 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002601
asaperssonfab67072017-04-04 05:51:49 -07002602 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002603 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002604 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002605 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002606 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002607 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002608 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002609 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002610
asaperssonfab67072017-04-04 05:51:49 -07002611 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002612 video_stream_encoder_->SetSource(&new_video_source,
2613 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002614
asaperssonfab67072017-04-04 05:51:49 -07002615 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002616 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002617 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002618 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002619 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002620 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002621
asapersson02465b82017-04-10 01:12:52 -07002622 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002623 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002624 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002625
asaperssonfab67072017-04-04 05:51:49 -07002626 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002627 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002628 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002629 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002630 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002631 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2632 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002633
mflodmancc3d4422017-08-03 08:27:51 -07002634 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002635}
2636
mflodmancc3d4422017-08-03 08:27:51 -07002637TEST_F(VideoStreamEncoderTest,
2638 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002640 DataRate::BitsPerSec(kTargetBitrateBps),
2641 DataRate::BitsPerSec(kTargetBitrateBps),
2642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002643
2644 const int kWidth = 1280;
2645 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002646 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002647 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002648 video_source_.IncomingCapturedFrame(
2649 CreateFrame(timestamp_ms, kWidth, kHeight));
2650 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2652 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2654
2655 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002656 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002657 timestamp_ms += kFrameIntervalMs;
2658 video_source_.IncomingCapturedFrame(
2659 CreateFrame(timestamp_ms, kWidth, kHeight));
2660 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002661 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2662 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2663 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2664
2665 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002666 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002667 timestamp_ms += kFrameIntervalMs;
2668 video_source_.IncomingCapturedFrame(
2669 CreateFrame(timestamp_ms, kWidth, kHeight));
2670 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002671 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2672 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2673 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2674
Niels Möller4db138e2018-04-19 09:04:13 +02002675 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002676 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002677
2678 VideoEncoderConfig video_encoder_config;
2679 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2680 // Make format different, to force recreation of encoder.
2681 video_encoder_config.video_format.parameters["foo"] = "foo";
2682 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002683 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002684 timestamp_ms += kFrameIntervalMs;
2685 video_source_.IncomingCapturedFrame(
2686 CreateFrame(timestamp_ms, kWidth, kHeight));
2687 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002688 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2690 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2691
mflodmancc3d4422017-08-03 08:27:51 -07002692 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002693}
2694
mflodmancc3d4422017-08-03 08:27:51 -07002695TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002696 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002698 DataRate::BitsPerSec(kTargetBitrateBps),
2699 DataRate::BitsPerSec(kTargetBitrateBps),
2700 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2701
2702 const int kWidth = 1280;
2703 const int kHeight = 720;
2704 int sequence = 1;
2705
2706 // Enable BALANCED preference, no initial limitation.
2707 test::FrameForwarder source;
2708 video_stream_encoder_->SetSource(&source,
2709 webrtc::DegradationPreference::BALANCED);
2710 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2711 WaitForEncodedFrame(sequence++);
2712 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2713 EXPECT_FALSE(stats.cpu_limited_resolution);
2714 EXPECT_FALSE(stats.cpu_limited_framerate);
2715 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2716
2717 // Trigger CPU overuse, should now adapt down.
2718 video_stream_encoder_->TriggerCpuOveruse();
2719 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2720 WaitForEncodedFrame(sequence++);
2721 stats = stats_proxy_->GetStats();
2722 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2723
2724 // Set new degradation preference should clear restrictions since we changed
2725 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002726 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002727 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2728 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2729 WaitForEncodedFrame(sequence++);
2730 stats = stats_proxy_->GetStats();
2731 EXPECT_FALSE(stats.cpu_limited_resolution);
2732 EXPECT_FALSE(stats.cpu_limited_framerate);
2733 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2734
2735 // Force an input frame rate to be available, or the adaptation call won't
2736 // know what framerate to adapt from.
2737 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2738 mock_stats.input_frame_rate = 30;
2739 stats_proxy_->SetMockStats(mock_stats);
2740 video_stream_encoder_->TriggerCpuOveruse();
2741 stats_proxy_->ResetMockStats();
2742 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2743 WaitForEncodedFrame(sequence++);
2744
2745 // We have now adapted once.
2746 stats = stats_proxy_->GetStats();
2747 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2748
2749 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002750 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2751 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002752 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2753 WaitForEncodedFrame(sequence++);
2754 stats = stats_proxy_->GetStats();
2755 EXPECT_FALSE(stats.cpu_limited_resolution);
2756 EXPECT_FALSE(stats.cpu_limited_framerate);
2757 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2758
2759 video_stream_encoder_->Stop();
2760}
2761
2762TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002763 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002764 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002765 DataRate::BitsPerSec(kTargetBitrateBps),
2766 DataRate::BitsPerSec(kTargetBitrateBps),
2767 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002768
asapersson0944a802017-04-07 00:57:58 -07002769 const int kWidth = 1280;
2770 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002771 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002772
asaperssonfab67072017-04-04 05:51:49 -07002773 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002774 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002775 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002776 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002777 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002778 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2779
asapersson02465b82017-04-10 01:12:52 -07002780 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002781 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002782 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002783 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002784 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002785 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002786 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002787 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2788
2789 // Set new source with adaptation still enabled.
2790 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002791 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002792 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002793
2794 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002795 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002796 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002797 stats = stats_proxy_->GetStats();
2798 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002799 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002800 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2801
sprangc5d62e22017-04-02 23:53:04 -07002802 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002803 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002804 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002805 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002806 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002807 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002808 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002809 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002810 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002811 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002812 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2813
sprangc5d62e22017-04-02 23:53:04 -07002814 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002815 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002816 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2817 mock_stats.input_frame_rate = 30;
2818 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002820 stats_proxy_->ResetMockStats();
2821
2822 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002823 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002824 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002825
2826 // Framerate now adapted.
2827 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002828 EXPECT_FALSE(stats.cpu_limited_resolution);
2829 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002830 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2831
2832 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002833 video_stream_encoder_->SetSource(&new_video_source,
2834 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002835 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002836 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002837 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002838
2839 stats = stats_proxy_->GetStats();
2840 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002841 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002842 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2843
2844 // Try to trigger overuse. Should not succeed.
2845 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002846 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002847 stats_proxy_->ResetMockStats();
2848
2849 stats = stats_proxy_->GetStats();
2850 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002851 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002852 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2853
2854 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002855 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002856 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002857 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002858 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002859 stats = stats_proxy_->GetStats();
2860 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002861 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002862 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002863
2864 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002865 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002866 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002867 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002868 stats = stats_proxy_->GetStats();
2869 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002870 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002871 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2872
2873 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002874 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002875 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002876 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002877 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002878 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002879 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002880 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002881 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002882 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002883 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2884
2885 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002886 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002887 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002888 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002889 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002890 stats = stats_proxy_->GetStats();
2891 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002892 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002893 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002894 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002895
mflodmancc3d4422017-08-03 08:27:51 -07002896 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002897}
2898
mflodmancc3d4422017-08-03 08:27:51 -07002899TEST_F(VideoStreamEncoderTest,
2900 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002901 const int kWidth = 1280;
2902 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002903 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002904 DataRate::BitsPerSec(kTargetBitrateBps),
2905 DataRate::BitsPerSec(kTargetBitrateBps),
2906 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002907
asaperssonfab67072017-04-04 05:51:49 -07002908 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002909 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002910
asaperssonfab67072017-04-04 05:51:49 -07002911 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002912 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002913
asaperssonfab67072017-04-04 05:51:49 -07002914 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002915 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002916
asaperssonfab67072017-04-04 05:51:49 -07002917 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002918 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002919
kthelgason876222f2016-11-29 01:44:11 -08002920 // Expect a scale down.
2921 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002922 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002923
asapersson02465b82017-04-10 01:12:52 -07002924 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002925 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002926 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002927 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002928
asaperssonfab67072017-04-04 05:51:49 -07002929 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002930 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002931 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002932 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002933
asaperssonfab67072017-04-04 05:51:49 -07002934 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002935 EXPECT_EQ(std::numeric_limits<int>::max(),
2936 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002937
asaperssonfab67072017-04-04 05:51:49 -07002938 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002939 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002940 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002941 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002942
asapersson02465b82017-04-10 01:12:52 -07002943 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002944 EXPECT_EQ(std::numeric_limits<int>::max(),
2945 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002946
mflodmancc3d4422017-08-03 08:27:51 -07002947 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002948}
2949
mflodmancc3d4422017-08-03 08:27:51 -07002950TEST_F(VideoStreamEncoderTest,
2951 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002952 const int kWidth = 1280;
2953 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002954 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002955 DataRate::BitsPerSec(kTargetBitrateBps),
2956 DataRate::BitsPerSec(kTargetBitrateBps),
2957 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002958
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002959 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002960 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002961 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002962 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002963
2964 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002965 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002966 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002967 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2968 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2969
2970 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002971 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002972 EXPECT_THAT(source.sink_wants(),
2973 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002974 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2975 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2976 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2977
2978 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002979 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002980 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2981 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2982 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2983
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002985}
2986
mflodmancc3d4422017-08-03 08:27:51 -07002987TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002988 const int kWidth = 1280;
2989 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002990 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002991 DataRate::BitsPerSec(kTargetBitrateBps),
2992 DataRate::BitsPerSec(kTargetBitrateBps),
2993 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002994
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002995 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002996 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002997 video_stream_encoder_->SetSource(&source,
2998 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002999 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3000 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003001 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003002
3003 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003004 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003005 EXPECT_THAT(source.sink_wants(),
3006 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003007 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3008 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3009 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3010
3011 // Trigger adapt down for same input resolution, expect no change.
3012 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3013 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003014 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003015 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3016 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3017 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3018
3019 // Trigger adapt down for larger input resolution, expect no change.
3020 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3021 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003022 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003023 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3024 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3025 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3026
mflodmancc3d4422017-08-03 08:27:51 -07003027 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003028}
3029
mflodmancc3d4422017-08-03 08:27:51 -07003030TEST_F(VideoStreamEncoderTest,
3031 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003032 const int kWidth = 1280;
3033 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003035 DataRate::BitsPerSec(kTargetBitrateBps),
3036 DataRate::BitsPerSec(kTargetBitrateBps),
3037 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003038
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003039 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003040 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003042 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003043
3044 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003045 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003046 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003047 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3048 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3049
3050 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003051 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003052 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003053 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3054 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3055
mflodmancc3d4422017-08-03 08:27:51 -07003056 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003057}
3058
mflodmancc3d4422017-08-03 08:27:51 -07003059TEST_F(VideoStreamEncoderTest,
3060 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003061 const int kWidth = 1280;
3062 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003064 DataRate::BitsPerSec(kTargetBitrateBps),
3065 DataRate::BitsPerSec(kTargetBitrateBps),
3066 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003067
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003068 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003069 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003070 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003071 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003072
3073 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003074 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003075 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003076 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003077 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3078
3079 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003080 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003081 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003082 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003083 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3084
mflodmancc3d4422017-08-03 08:27:51 -07003085 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003086}
3087
mflodmancc3d4422017-08-03 08:27:51 -07003088TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003089 const int kWidth = 1280;
3090 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003092 DataRate::BitsPerSec(kTargetBitrateBps),
3093 DataRate::BitsPerSec(kTargetBitrateBps),
3094 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003095
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003096 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003097 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003098 video_stream_encoder_->SetSource(&source,
3099 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003100
3101 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3102 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003103 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003104 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3105 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3106 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3107
3108 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003109 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003110 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003111 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3112 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3113 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3114
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003116}
3117
mflodmancc3d4422017-08-03 08:27:51 -07003118TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003119 const int kWidth = 1280;
3120 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003122 DataRate::BitsPerSec(kTargetBitrateBps),
3123 DataRate::BitsPerSec(kTargetBitrateBps),
3124 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003125
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003126 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003127 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003128 video_stream_encoder_->SetSource(&source,
3129 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003130
3131 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3132 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003133 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003134 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3135 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3136 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3137
3138 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003139 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003140 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003141 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3142 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3143 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3144
mflodmancc3d4422017-08-03 08:27:51 -07003145 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003146}
3147
mflodmancc3d4422017-08-03 08:27:51 -07003148TEST_F(VideoStreamEncoderTest,
3149 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003150 const int kWidth = 1280;
3151 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003152 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003153 DataRate::BitsPerSec(kTargetBitrateBps),
3154 DataRate::BitsPerSec(kTargetBitrateBps),
3155 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003156
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003157 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003158 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003159 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003160 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003161 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003162
3163 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003164 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003165 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003166 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3167 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3168
3169 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003170 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003171 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003172 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003173 EXPECT_THAT(source.sink_wants(),
3174 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003175 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3176 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3177
3178 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003179 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003180 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003181 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3182 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3183 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3184
mflodmancc3d4422017-08-03 08:27:51 -07003185 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003186}
3187
mflodmancc3d4422017-08-03 08:27:51 -07003188TEST_F(VideoStreamEncoderTest,
3189 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003190 const int kWidth = 1280;
3191 const int kHeight = 720;
3192 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003194 DataRate::BitsPerSec(kTargetBitrateBps),
3195 DataRate::BitsPerSec(kTargetBitrateBps),
3196 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003197
3198 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3199 stats.input_frame_rate = kInputFps;
3200 stats_proxy_->SetMockStats(stats);
3201
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003202 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003203 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3204 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003205 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003206
3207 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003208 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003209 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3210 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003211 EXPECT_THAT(video_source_.sink_wants(),
3212 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003213
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003214 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003215 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003216 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003217 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003218 // Give the encoder queue time to process the change in degradation preference
3219 // by waiting for an encoded frame.
3220 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3221 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003222 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003223
3224 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003225 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003226 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3227 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003228 EXPECT_THAT(new_video_source.sink_wants(),
3229 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003230
3231 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003232 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003233 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003234
mflodmancc3d4422017-08-03 08:27:51 -07003235 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003236}
3237
mflodmancc3d4422017-08-03 08:27:51 -07003238TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003239 const int kWidth = 1280;
3240 const int kHeight = 720;
3241 const size_t kNumFrames = 10;
3242
Henrik Boström381d1092020-05-12 18:49:07 +02003243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003244 DataRate::BitsPerSec(kTargetBitrateBps),
3245 DataRate::BitsPerSec(kTargetBitrateBps),
3246 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003247
asaperssond0de2952017-04-21 01:47:31 -07003248 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003249 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003250 video_source_.set_adaptation_enabled(true);
3251
3252 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3253 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3254
3255 int downscales = 0;
3256 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003257 video_source_.IncomingCapturedFrame(
3258 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3259 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003260
asaperssonfab67072017-04-04 05:51:49 -07003261 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003262 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003263 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003264 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003265
3266 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3267 ++downscales;
3268
3269 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3270 EXPECT_EQ(downscales,
3271 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3272 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003273 }
mflodmancc3d4422017-08-03 08:27:51 -07003274 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003275}
3276
mflodmancc3d4422017-08-03 08:27:51 -07003277TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003278 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3279 const int kWidth = 1280;
3280 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003281 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003282 DataRate::BitsPerSec(kTargetBitrateBps),
3283 DataRate::BitsPerSec(kTargetBitrateBps),
3284 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003285
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003286 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003287 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003288 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003289 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003290 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003291
Åsa Persson8c1bf952018-09-13 10:42:19 +02003292 int64_t timestamp_ms = kFrameIntervalMs;
3293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003294 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003295 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003296 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3297 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3298
3299 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003301 timestamp_ms += kFrameIntervalMs;
3302 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3303 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003304 EXPECT_THAT(source.sink_wants(),
3305 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003306 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3307 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3308
3309 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003310 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003311 timestamp_ms += kFrameIntervalMs;
3312 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003313 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003314 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003315 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3316 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3317
3318 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003319 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003320 timestamp_ms += kFrameIntervalMs;
3321 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3322 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003323 EXPECT_THAT(source.sink_wants(),
3324 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003325 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3326 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3327
3328 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003329 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003330 timestamp_ms += kFrameIntervalMs;
3331 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003332 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003333 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003334 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3335 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3336
mflodmancc3d4422017-08-03 08:27:51 -07003337 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003338}
3339
mflodmancc3d4422017-08-03 08:27:51 -07003340TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003341 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3342 const int kWidth = 1280;
3343 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003344 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003345 DataRate::BitsPerSec(kTargetBitrateBps),
3346 DataRate::BitsPerSec(kTargetBitrateBps),
3347 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003348
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003349 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003350 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003351 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003352 video_stream_encoder_->SetSource(&source,
3353 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003354
Åsa Persson8c1bf952018-09-13 10:42:19 +02003355 int64_t timestamp_ms = kFrameIntervalMs;
3356 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003357 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003358 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003359 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3360 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3361
3362 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003363 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003364 timestamp_ms += kFrameIntervalMs;
3365 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3366 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003367 EXPECT_THAT(source.sink_wants(),
3368 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003369 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3370 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3371
3372 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003373 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003374 timestamp_ms += kFrameIntervalMs;
3375 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003376 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003377 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003378 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3379 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3380
3381 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003383 timestamp_ms += kFrameIntervalMs;
3384 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3385 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003386 EXPECT_THAT(source.sink_wants(),
3387 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003388 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3389 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3390
3391 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003393 timestamp_ms += kFrameIntervalMs;
3394 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003395 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003396 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003397 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3398 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
mflodmancc3d4422017-08-03 08:27:51 -07003400 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003401}
3402
Sergey Silkin41c650b2019-10-14 13:12:19 +02003403TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3404 fake_encoder_.SetResolutionBitrateLimits(
3405 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3406
Henrik Boström381d1092020-05-12 18:49:07 +02003407 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003408 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3409 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3410 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3411 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003412
3413 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003414 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003415 source.set_adaptation_enabled(true);
3416 video_stream_encoder_->SetSource(
3417 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3418
3419 // Insert 720p frame.
3420 int64_t timestamp_ms = kFrameIntervalMs;
3421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3422 WaitForEncodedFrame(1280, 720);
3423
3424 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003425 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003426 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3427 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3428 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3429 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003430 video_stream_encoder_->TriggerQualityLow();
3431
3432 // Insert 720p frame. It should be downscaled and encoded.
3433 timestamp_ms += kFrameIntervalMs;
3434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3435 WaitForEncodedFrame(960, 540);
3436
3437 // Trigger adapt up. Higher resolution should not be requested duo to lack
3438 // of bitrate.
3439 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003440 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003441
3442 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003444 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3445 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3446 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3447 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003448
3449 // Trigger adapt up. Higher resolution should be requested.
3450 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003451 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003452
3453 video_stream_encoder_->Stop();
3454}
3455
3456TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3457 fake_encoder_.SetResolutionBitrateLimits(
3458 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3459
3460 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003461 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003462 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3463 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3464 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3465 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003466
3467 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003468 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003469 source.set_adaptation_enabled(true);
3470 video_stream_encoder_->SetSource(
3471 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3472
3473 // Insert 720p frame. It should be dropped and lower resolution should be
3474 // requested.
3475 int64_t timestamp_ms = kFrameIntervalMs;
3476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3477 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003478 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003479
3480 // Insert 720p frame. It should be downscaled and encoded.
3481 timestamp_ms += kFrameIntervalMs;
3482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3483 WaitForEncodedFrame(960, 540);
3484
3485 video_stream_encoder_->Stop();
3486}
3487
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003488class BalancedDegradationTest : public VideoStreamEncoderTest {
3489 protected:
3490 void SetupTest() {
3491 // Reset encoder for field trials to take effect.
3492 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003493 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003494
3495 // Enable BALANCED preference.
3496 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003497 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3498 }
3499
3500 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003502 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3503 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003504 }
3505
Åsa Persson45b176f2019-09-30 11:19:05 +02003506 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003507 timestamp_ms_ += kFrameIntervalMs;
3508 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003509 }
3510
3511 void InsertFrameAndWaitForEncoded() {
3512 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003513 sink_.WaitForEncodedFrame(timestamp_ms_);
3514 }
3515
3516 const int kWidth = 640; // pixels:640x360=230400
3517 const int kHeight = 360;
3518 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3519 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003520 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003521};
3522
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003523TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003524 test::ScopedFieldTrials field_trials(
3525 "WebRTC-Video-BalancedDegradationSettings/"
3526 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3527 SetupTest();
3528
3529 // Force input frame rate.
3530 const int kInputFps = 24;
3531 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3532 stats.input_frame_rate = kInputFps;
3533 stats_proxy_->SetMockStats(stats);
3534
Åsa Persson45b176f2019-09-30 11:19:05 +02003535 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003536 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003537
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003538 // Trigger adapt down, expect scaled down framerate and resolution,
3539 // since Fps diff (input-requested:0) < threshold.
3540 video_stream_encoder_->TriggerQualityLow();
3541 EXPECT_THAT(source_.sink_wants(),
3542 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003543
3544 video_stream_encoder_->Stop();
3545}
3546
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003547TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003548 test::ScopedFieldTrials field_trials(
3549 "WebRTC-Video-BalancedDegradationSettings/"
3550 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3551 SetupTest();
3552
3553 // Force input frame rate.
3554 const int kInputFps = 25;
3555 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3556 stats.input_frame_rate = kInputFps;
3557 stats_proxy_->SetMockStats(stats);
3558
Åsa Persson45b176f2019-09-30 11:19:05 +02003559 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003560 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003561
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003562 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3563 // Fps diff (input-requested:1) == threshold.
3564 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003565 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003566
3567 video_stream_encoder_->Stop();
3568}
3569
3570TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3571 test::ScopedFieldTrials field_trials(
3572 "WebRTC-Video-BalancedDegradationSettings/"
3573 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3574 SetupTest();
3575
3576 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3577
Åsa Persson45b176f2019-09-30 11:19:05 +02003578 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003579 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003580
3581 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3582 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003583 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003584
3585 video_stream_encoder_->Stop();
3586}
3587
Åsa Perssonccfb3402019-09-25 15:13:04 +02003588TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003589 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003590 "WebRTC-Video-BalancedDegradationSettings/"
3591 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003592 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003593
Åsa Persson1b247f12019-08-14 17:26:39 +02003594 const int kMinBitrateBps = 425000;
3595 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003596 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003597
Åsa Persson45b176f2019-09-30 11:19:05 +02003598 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003599 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003600 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3601
3602 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3603 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003604 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003605 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003606 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3607
3608 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3609 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003610 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003611 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003612 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3613
Åsa Persson30ab0152019-08-27 12:22:33 +02003614 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3615 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003616 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003617 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003618 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003619 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3620
3621 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003622 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003623 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003624 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003625
Åsa Persson30ab0152019-08-27 12:22:33 +02003626 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003627 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003628 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003629 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003630 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003631 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3632
3633 video_stream_encoder_->Stop();
3634}
3635
Åsa Perssonccfb3402019-09-25 15:13:04 +02003636TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003637 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3638 test::ScopedFieldTrials field_trials(
3639 "WebRTC-Video-BalancedDegradationSettings/"
3640 "pixels:57600|129600|230400,fps:7|24|24/");
3641 SetupTest();
3642 OnBitrateUpdated(kLowTargetBitrateBps);
3643
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003644 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003645
3646 // Insert frame, expect scaled down:
3647 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3648 InsertFrame();
3649 EXPECT_FALSE(WaitForFrame(1000));
3650 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3651 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3652
3653 // Insert frame, expect scaled down:
3654 // resolution (320x180@24fps).
3655 InsertFrame();
3656 EXPECT_FALSE(WaitForFrame(1000));
3657 EXPECT_LT(source_.sink_wants().max_pixel_count,
3658 source_.last_wants().max_pixel_count);
3659 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3660
3661 // Frame should not be dropped (min pixels per frame reached).
3662 InsertFrameAndWaitForEncoded();
3663
3664 video_stream_encoder_->Stop();
3665}
3666
3667TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003668 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003669 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003670 "WebRTC-Video-BalancedDegradationSettings/"
3671 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003672 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003673
Åsa Persson30ab0152019-08-27 12:22:33 +02003674 const int kResolutionMinBitrateBps = 435000;
3675 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003676 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003677
Åsa Persson45b176f2019-09-30 11:19:05 +02003678 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003679 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003680 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3681
3682 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3683 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003684 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003685 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003686 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3687
3688 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3689 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003690 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003691 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003692 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3693
3694 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3695 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003696 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003697 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003698 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3699
Åsa Persson30ab0152019-08-27 12:22:33 +02003700 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3701 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003702 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003703 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003704 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3705
3706 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3707 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003708 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003709 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3710
3711 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003712 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003713 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003714 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003715 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003716 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3717
3718 video_stream_encoder_->Stop();
3719}
3720
Åsa Perssonccfb3402019-09-25 15:13:04 +02003721TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003722 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003723 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003724 "WebRTC-Video-BalancedDegradationSettings/"
3725 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003726 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003727
Åsa Persson30ab0152019-08-27 12:22:33 +02003728 const int kMinBitrateBps = 425000;
3729 const int kTooLowMinBitrateBps = 424000;
3730 const int kResolutionMinBitrateBps = 435000;
3731 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003732 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003733
Åsa Persson45b176f2019-09-30 11:19:05 +02003734 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003735 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003736 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3737
3738 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3739 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003740 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003741 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003742 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3743
3744 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3745 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003746 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003747 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003748 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3749
3750 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3751 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003752 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003753 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003754 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3755
3756 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3757 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003758 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003759 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3760
3761 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003762 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003763 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003764 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003765 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003766 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3767
3768 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003769 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003770 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003771 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003772 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3773
3774 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003775 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003776 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003777 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003778 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003779 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3780
Åsa Persson1b247f12019-08-14 17:26:39 +02003781 video_stream_encoder_->Stop();
3782}
3783
mflodmancc3d4422017-08-03 08:27:51 -07003784TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003785 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3786 const int kWidth = 1280;
3787 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003789 DataRate::BitsPerSec(kTargetBitrateBps),
3790 DataRate::BitsPerSec(kTargetBitrateBps),
3791 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003792
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003793 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003794 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003795 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003796 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003797 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003798
Åsa Persson8c1bf952018-09-13 10:42:19 +02003799 int64_t timestamp_ms = kFrameIntervalMs;
3800 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003801 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003802 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003803 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3804 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3805 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3806 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3807
3808 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003809 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003810 timestamp_ms += kFrameIntervalMs;
3811 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3812 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003813 EXPECT_THAT(source.sink_wants(),
3814 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003815 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3817 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3818 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3819
3820 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003821 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003822 timestamp_ms += kFrameIntervalMs;
3823 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3824 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003825 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003826 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3828 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3830
Jonathan Yubc771b72017-12-08 17:04:29 -08003831 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003832 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003833 timestamp_ms += kFrameIntervalMs;
3834 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3835 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003836 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003837 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3838 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003839 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003840 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3841
Jonathan Yubc771b72017-12-08 17:04:29 -08003842 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003843 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003844 timestamp_ms += kFrameIntervalMs;
3845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3846 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003847 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003848 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003849 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3850 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3851 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3852 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3853
Jonathan Yubc771b72017-12-08 17:04:29 -08003854 // Trigger quality adapt down, expect no change (min resolution reached).
3855 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003856 timestamp_ms += kFrameIntervalMs;
3857 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3858 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003859 EXPECT_THAT(source.sink_wants(), FpsMax());
3860 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003861 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3862 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3863 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3864 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3865
Evan Shrubsole64469032020-06-11 10:45:29 +02003866 // Trigger quality adapt up, expect upscaled resolution (480x270).
3867 video_stream_encoder_->TriggerQualityHigh();
3868 timestamp_ms += kFrameIntervalMs;
3869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3870 WaitForEncodedFrame(timestamp_ms);
3871 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3872 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3873 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3874 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3875 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3876
3877 // Trigger quality and cpu adapt up since both are most limited, expect
3878 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003879 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003880 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003881 timestamp_ms += kFrameIntervalMs;
3882 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3883 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003884 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003885 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3886 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3887 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003888 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003889
Evan Shrubsole64469032020-06-11 10:45:29 +02003890 // Trigger quality and cpu adapt up since both are most limited, expect
3891 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003892 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003893 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003894 timestamp_ms += kFrameIntervalMs;
3895 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3896 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003897 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003898 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003899 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003900 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003901 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3902 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003903
Evan Shrubsole64469032020-06-11 10:45:29 +02003904 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3905 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003906 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003907 timestamp_ms += kFrameIntervalMs;
3908 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3909 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003910 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003911 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3912 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003913 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003914 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003915
3916 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003917 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003918 timestamp_ms += kFrameIntervalMs;
3919 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003920 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003921 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003922 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003923 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3924 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003925 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003926 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003927
mflodmancc3d4422017-08-03 08:27:51 -07003928 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003929}
3930
mflodmancc3d4422017-08-03 08:27:51 -07003931TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003932 const int kWidth = 640;
3933 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003934
Henrik Boström381d1092020-05-12 18:49:07 +02003935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003936 DataRate::BitsPerSec(kTargetBitrateBps),
3937 DataRate::BitsPerSec(kTargetBitrateBps),
3938 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003939
perkj803d97f2016-11-01 11:45:46 -07003940 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003941 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003942 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003943 }
3944
mflodmancc3d4422017-08-03 08:27:51 -07003945 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003946 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003947 video_source_.IncomingCapturedFrame(CreateFrame(
3948 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003949 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003950 }
3951
mflodmancc3d4422017-08-03 08:27:51 -07003952 video_stream_encoder_->Stop();
3953 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003954 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003955
Ying Wangef3998f2019-12-09 13:06:53 +01003956 EXPECT_METRIC_EQ(
3957 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3958 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003959 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3960}
3961
mflodmancc3d4422017-08-03 08:27:51 -07003962TEST_F(VideoStreamEncoderTest,
3963 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003965 DataRate::BitsPerSec(kTargetBitrateBps),
3966 DataRate::BitsPerSec(kTargetBitrateBps),
3967 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003968 const int kWidth = 640;
3969 const int kHeight = 360;
3970
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003971 video_stream_encoder_->SetSource(&video_source_,
3972 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003973
3974 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3975 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003976 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003977 }
3978
mflodmancc3d4422017-08-03 08:27:51 -07003979 video_stream_encoder_->Stop();
3980 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003981 stats_proxy_.reset();
3982
3983 EXPECT_EQ(0,
3984 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3985}
3986
Per Kjellanderdcef6412020-10-07 15:09:05 +02003987TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
3988 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
3989 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
3990 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08003991
3992 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003993 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003994 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003995 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3996 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003997
Henrik Boström381d1092020-05-12 18:49:07 +02003998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003999 DataRate::BitsPerSec(kLowTargetBitrateBps),
4000 DataRate::BitsPerSec(kLowTargetBitrateBps),
4001 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004002
sprang57c2fff2017-01-16 06:24:02 -08004003 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004004 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4005 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004006 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4007 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4008
Erik Språngd7329ca2019-02-21 21:19:53 +01004009 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004010 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004011 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004012
Per Kjellanderdcef6412020-10-07 15:09:05 +02004013 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004014 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004015 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4016 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004017 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004018 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004019
Per Kjellanderdcef6412020-10-07 15:09:05 +02004020 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004021 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004022 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004023 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004024 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4025 WaitForEncodedFrame(CurrentTimeMs());
4026 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004027 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004028 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004029
mflodmancc3d4422017-08-03 08:27:51 -07004030 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004031}
4032
Per Kjellandera9434842020-10-15 17:53:22 +02004033TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForV8Simulcast) {
4034 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
4035 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4036 kVideoLayersAllocation);
4037
4038 const int kDefaultFps = 30;
4039
4040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4041 DataRate::BitsPerSec(kLowTargetBitrateBps),
4042 DataRate::BitsPerSec(kLowTargetBitrateBps),
4043 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4044
4045 video_source_.IncomingCapturedFrame(
4046 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4047 WaitForEncodedFrame(CurrentTimeMs());
4048 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4049 VideoLayersAllocation last_layer_allocation =
4050 sink_.GetLastVideoLayersAllocation();
4051 // kLowTargetBitrateBps is only enough for one spatial layer.
4052 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4053
4054 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004055 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004056 // Check that encoder has been updated too, not just allocation observer.
4057 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4058 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4059
Erik Språng9d69cbe2020-10-22 17:44:42 +02004060 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004061 int number_of_layers_allocation = 1;
4062 const int64_t start_time_ms = CurrentTimeMs();
4063 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4064 video_source_.IncomingCapturedFrame(
4065 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4066 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004067 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4068 number_of_layers_allocation = sink_.number_of_layers_allocations();
4069 VideoLayersAllocation new_allocation =
4070 sink_.GetLastVideoLayersAllocation();
4071 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4072 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4073 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4074 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4075 .target_bitrate_per_temporal_layer,
4076 last_layer_allocation.active_spatial_layers[0]
4077 .target_bitrate_per_temporal_layer);
4078 last_layer_allocation = new_allocation;
4079 }
4080 }
4081 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4082 video_stream_encoder_->Stop();
4083}
4084
4085TEST_F(VideoStreamEncoderTest,
4086 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4087 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
4088 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4089 kVideoLayersAllocation);
4090
4091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4092 DataRate::BitsPerSec(kLowTargetBitrateBps),
4093 DataRate::BitsPerSec(kLowTargetBitrateBps),
4094 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4095
4096 video_source_.IncomingCapturedFrame(
4097 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4098 WaitForEncodedFrame(CurrentTimeMs());
4099 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4100 VideoLayersAllocation last_layer_allocation =
4101 sink_.GetLastVideoLayersAllocation();
4102 // kLowTargetBitrateBps is only enough for one spatial layer.
4103 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4104 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4105 .target_bitrate_per_temporal_layer[0],
4106 DataRate::BitsPerSec(kLowTargetBitrateBps));
4107
4108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4109 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4110 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4111 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4112 video_source_.IncomingCapturedFrame(
4113 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4114 WaitForEncodedFrame(CurrentTimeMs());
4115
4116 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4117 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4118 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4119 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4120 .target_bitrate_per_temporal_layer[0],
4121 DataRate::Zero());
4122
4123 video_stream_encoder_->Stop();
4124}
4125
Per Kjellander4190ce92020-12-15 17:24:55 +01004126TEST_F(VideoStreamEncoderTest,
4127 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4128 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
4129 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4130 kVideoLayersAllocation);
4131
4132 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4133 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4134 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4135 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4136
4137 video_source_.IncomingCapturedFrame(
4138 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4139 WaitForEncodedFrame(CurrentTimeMs());
4140 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4141 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4142 SizeIs(2));
4143 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4144 codec_width_);
4145 EXPECT_EQ(
4146 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4147 codec_height_);
4148
4149 video_source_.IncomingCapturedFrame(
4150 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4151 WaitForEncodedFrame(CurrentTimeMs());
4152 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4153 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4154 SizeIs(2));
4155 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4156 codec_width_ / 2);
4157 EXPECT_EQ(
4158 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4159 codec_height_ / 2);
4160
4161 video_stream_encoder_->Stop();
4162}
4163
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004164TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4165 // 2 TLs configured, temporal layers supported by encoder.
4166 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004167 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
4168 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4169 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004170 fake_encoder_.SetTemporalLayersSupported(0, true);
4171
4172 // Bitrate allocated across temporal layers.
4173 const int kTl0Bps = kTargetBitrateBps *
4174 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004175 kNumTemporalLayers, /*temporal_id*/ 0,
4176 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004177 const int kTl1Bps = kTargetBitrateBps *
4178 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004179 kNumTemporalLayers, /*temporal_id*/ 1,
4180 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004181 VideoBitrateAllocation expected_bitrate;
4182 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4183 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4184
4185 VerifyAllocatedBitrate(expected_bitrate);
4186 video_stream_encoder_->Stop();
4187}
4188
4189TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4190 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004191 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
4192 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4193 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004194 fake_encoder_.SetTemporalLayersSupported(0, false);
4195
4196 // Temporal layers not supported by the encoder.
4197 // Total bitrate should be at ti:0.
4198 VideoBitrateAllocation expected_bitrate;
4199 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4200
4201 VerifyAllocatedBitrate(expected_bitrate);
4202 video_stream_encoder_->Stop();
4203}
4204
4205TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004206 webrtc::test::ScopedFieldTrials field_trials(
4207 "WebRTC-Video-QualityScalerSettings/"
4208 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4209 // Reset encoder for field trials to take effect.
4210 ConfigureEncoder(video_encoder_config_.Copy());
4211
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004212 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004213 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
4214 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4215 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004216 fake_encoder_.SetTemporalLayersSupported(0, true);
4217 fake_encoder_.SetTemporalLayersSupported(1, false);
4218
4219 const int kS0Bps = 150000;
4220 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004221 kS0Bps *
4222 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4223 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004224 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004225 kS0Bps *
4226 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4227 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004228 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4229 // Temporal layers not supported by si:1.
4230 VideoBitrateAllocation expected_bitrate;
4231 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4232 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4233 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4234
4235 VerifyAllocatedBitrate(expected_bitrate);
4236 video_stream_encoder_->Stop();
4237}
4238
Niels Möller7dc26b72017-12-06 10:27:48 +01004239TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4240 const int kFrameWidth = 1280;
4241 const int kFrameHeight = 720;
4242 const int kFramerate = 24;
4243
Henrik Boström381d1092020-05-12 18:49:07 +02004244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004245 DataRate::BitsPerSec(kTargetBitrateBps),
4246 DataRate::BitsPerSec(kTargetBitrateBps),
4247 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004248 test::FrameForwarder source;
4249 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004250 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004251
4252 // Insert a single frame, triggering initial configuration.
4253 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4254 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4255
4256 EXPECT_EQ(
4257 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4258 kDefaultFramerate);
4259
4260 // Trigger reconfigure encoder (without resetting the entire instance).
4261 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004262 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4263 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004264 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004265 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004266 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004267 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4268
4269 // Detector should be updated with fps limit from codec config.
4270 EXPECT_EQ(
4271 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4272 kFramerate);
4273
4274 // Trigger overuse, max framerate should be reduced.
4275 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4276 stats.input_frame_rate = kFramerate;
4277 stats_proxy_->SetMockStats(stats);
4278 video_stream_encoder_->TriggerCpuOveruse();
4279 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4280 int adapted_framerate =
4281 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4282 EXPECT_LT(adapted_framerate, kFramerate);
4283
4284 // Trigger underuse, max framerate should go back to codec configured fps.
4285 // Set extra low fps, to make sure it's actually reset, not just incremented.
4286 stats = stats_proxy_->GetStats();
4287 stats.input_frame_rate = adapted_framerate / 2;
4288 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004289 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4291 EXPECT_EQ(
4292 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4293 kFramerate);
4294
4295 video_stream_encoder_->Stop();
4296}
4297
4298TEST_F(VideoStreamEncoderTest,
4299 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
4300 const int kFrameWidth = 1280;
4301 const int kFrameHeight = 720;
4302 const int kLowFramerate = 15;
4303 const int kHighFramerate = 25;
4304
Henrik Boström381d1092020-05-12 18:49:07 +02004305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004306 DataRate::BitsPerSec(kTargetBitrateBps),
4307 DataRate::BitsPerSec(kTargetBitrateBps),
4308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004309 test::FrameForwarder source;
4310 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004311 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004312
4313 // Trigger initial configuration.
4314 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004315 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4316 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004317 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004318 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02004319 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02004320 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004321 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4322
4323 EXPECT_EQ(
4324 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4325 kLowFramerate);
4326
4327 // Trigger overuse, max framerate should be reduced.
4328 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4329 stats.input_frame_rate = kLowFramerate;
4330 stats_proxy_->SetMockStats(stats);
4331 video_stream_encoder_->TriggerCpuOveruse();
4332 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4333 int adapted_framerate =
4334 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4335 EXPECT_LT(adapted_framerate, kLowFramerate);
4336
4337 // Reconfigure the encoder with a new (higher max framerate), max fps should
4338 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02004339 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004340 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4341 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004342 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004343 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4344
4345 EXPECT_EQ(
4346 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4347 adapted_framerate);
4348
4349 // Trigger underuse, max framerate should go back to codec configured fps.
4350 stats = stats_proxy_->GetStats();
4351 stats.input_frame_rate = adapted_framerate;
4352 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004353 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004354 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4355 EXPECT_EQ(
4356 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4357 kHighFramerate);
4358
4359 video_stream_encoder_->Stop();
4360}
4361
mflodmancc3d4422017-08-03 08:27:51 -07004362TEST_F(VideoStreamEncoderTest,
4363 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07004364 const int kFrameWidth = 1280;
4365 const int kFrameHeight = 720;
4366 const int kFramerate = 24;
4367
Henrik Boström381d1092020-05-12 18:49:07 +02004368 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004369 DataRate::BitsPerSec(kTargetBitrateBps),
4370 DataRate::BitsPerSec(kTargetBitrateBps),
4371 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07004372 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004373 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004374 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07004375
4376 // Trigger initial configuration.
4377 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004378 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4379 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07004380 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07004381 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004382 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004383 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004384 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004385
Niels Möller7dc26b72017-12-06 10:27:48 +01004386 EXPECT_EQ(
4387 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4388 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004389
4390 // Trigger overuse, max framerate should be reduced.
4391 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4392 stats.input_frame_rate = kFramerate;
4393 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004394 video_stream_encoder_->TriggerCpuOveruse();
4395 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004396 int adapted_framerate =
4397 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004398 EXPECT_LT(adapted_framerate, kFramerate);
4399
4400 // Change degradation preference to not enable framerate scaling. Target
4401 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004402 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004403 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004404 EXPECT_EQ(
4405 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4406 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004407
mflodmancc3d4422017-08-03 08:27:51 -07004408 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004409}
4410
mflodmancc3d4422017-08-03 08:27:51 -07004411TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004412 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004414 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4415 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4416 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004417 const int kWidth = 640;
4418 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004419
asaperssonfab67072017-04-04 05:51:49 -07004420 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004421
4422 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004423 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004424
4425 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004426 EXPECT_TRUE_WAIT(
4427 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004428
sprangc5d62e22017-04-02 23:53:04 -07004429 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004430
asaperssonfab67072017-04-04 05:51:49 -07004431 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004432 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004433 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004434
4435 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004436 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004437
Henrik Boström2671dac2020-05-19 16:29:09 +02004438 EXPECT_TRUE_WAIT(
4439 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004440
mflodmancc3d4422017-08-03 08:27:51 -07004441 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004442}
4443
mflodmancc3d4422017-08-03 08:27:51 -07004444TEST_F(VideoStreamEncoderTest,
4445 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004446 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004448 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4449 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4450 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004451 const int kWidth = 640;
4452 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004453
4454 // We expect the n initial frames to get dropped.
4455 int i;
4456 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004457 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004458 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004459 }
4460 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004461 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004462 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004463
4464 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004465 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004466
mflodmancc3d4422017-08-03 08:27:51 -07004467 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004468}
4469
mflodmancc3d4422017-08-03 08:27:51 -07004470TEST_F(VideoStreamEncoderTest,
4471 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004472 const int kWidth = 640;
4473 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004475 DataRate::BitsPerSec(kLowTargetBitrateBps),
4476 DataRate::BitsPerSec(kLowTargetBitrateBps),
4477 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004478
4479 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004480 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004481 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004482
asaperssonfab67072017-04-04 05:51:49 -07004483 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004484 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004485 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004486
mflodmancc3d4422017-08-03 08:27:51 -07004487 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004488}
4489
mflodmancc3d4422017-08-03 08:27:51 -07004490TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004491 const int kWidth = 640;
4492 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004493 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004494
4495 VideoEncoderConfig video_encoder_config;
4496 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4497 // Make format different, to force recreation of encoder.
4498 video_encoder_config.video_format.parameters["foo"] = "foo";
4499 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004500 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004502 DataRate::BitsPerSec(kLowTargetBitrateBps),
4503 DataRate::BitsPerSec(kLowTargetBitrateBps),
4504 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004505
kthelgasonb83797b2017-02-14 11:57:25 -08004506 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004507 video_stream_encoder_->SetSource(&video_source_,
4508 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004509
asaperssonfab67072017-04-04 05:51:49 -07004510 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004511 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004512 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004513
mflodmancc3d4422017-08-03 08:27:51 -07004514 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004515 fake_encoder_.SetQualityScaling(true);
4516}
4517
Åsa Persson139f4dc2019-08-02 09:29:58 +02004518TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4519 webrtc::test::ScopedFieldTrials field_trials(
4520 "WebRTC-Video-QualityScalerSettings/"
4521 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4522 // Reset encoder for field trials to take effect.
4523 ConfigureEncoder(video_encoder_config_.Copy());
4524 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4525 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4526 const int kWidth = 640;
4527 const int kHeight = 360;
4528
Henrik Boström381d1092020-05-12 18:49:07 +02004529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004530 DataRate::BitsPerSec(kTargetBitrateBps),
4531 DataRate::BitsPerSec(kTargetBitrateBps),
4532 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004533 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4534 // Frame should not be dropped.
4535 WaitForEncodedFrame(1);
4536
Henrik Boström381d1092020-05-12 18:49:07 +02004537 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004538 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4539 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4540 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004541 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4542 // Frame should not be dropped.
4543 WaitForEncodedFrame(2);
4544
Henrik Boström381d1092020-05-12 18:49:07 +02004545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004546 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4547 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4548 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004549 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4550 // Expect to drop this frame, the wait should time out.
4551 ExpectDroppedFrame();
4552
4553 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004554 EXPECT_TRUE_WAIT(
4555 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004556 video_stream_encoder_->Stop();
4557}
4558
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004559TEST_F(VideoStreamEncoderTest,
4560 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
4561 webrtc::test::ScopedFieldTrials field_trials(
4562 "WebRTC-Video-QualityScalerSettings/"
4563 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4564 fake_encoder_.SetQualityScaling(false);
4565 ConfigureEncoder(video_encoder_config_.Copy());
4566 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4567 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4568 const int kWidth = 640;
4569 const int kHeight = 360;
4570
4571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4572 DataRate::BitsPerSec(kTargetBitrateBps),
4573 DataRate::BitsPerSec(kTargetBitrateBps),
4574 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4575 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4576 // Frame should not be dropped.
4577 WaitForEncodedFrame(1);
4578
4579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4580 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4581 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4582 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
4583 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4584 // Frame should not be dropped.
4585 WaitForEncodedFrame(2);
4586
4587 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4588 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4589 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4590 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
4591 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4592 // Not dropped since quality scaling is disabled.
4593 WaitForEncodedFrame(3);
4594
4595 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02004596 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004597 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
4598
4599 video_stream_encoder_->Stop();
4600}
4601
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01004602TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
4603 const int kLowTargetBitrateBps = 400000;
4604 // Set simulcast.
4605 ResetEncoder("VP8", 3, 1, 1, false);
4606 fake_encoder_.SetQualityScaling(true);
4607 const int kWidth = 1280;
4608 const int kHeight = 720;
4609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4610 DataRate::BitsPerSec(kLowTargetBitrateBps),
4611 DataRate::BitsPerSec(kLowTargetBitrateBps),
4612 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4613 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4614 // Frame should not be dropped.
4615 WaitForEncodedFrame(1);
4616
4617 // Trigger QVGA "singlecast"
4618 // Update the config.
4619 VideoEncoderConfig video_encoder_config;
4620 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
4621 &video_encoder_config);
4622 for (auto& layer : video_encoder_config.simulcast_layers) {
4623 layer.num_temporal_layers = 1;
4624 layer.max_framerate = kDefaultFramerate;
4625 }
4626 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
4627 video_encoder_config.content_type =
4628 VideoEncoderConfig::ContentType::kRealtimeVideo;
4629
4630 video_encoder_config.simulcast_layers[0].active = true;
4631 video_encoder_config.simulcast_layers[1].active = false;
4632 video_encoder_config.simulcast_layers[2].active = false;
4633
4634 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
4635 kMaxPayloadLength);
4636 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4637
4638 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4639 // Frame should not be dropped.
4640 WaitForEncodedFrame(2);
4641
4642 // Trigger HD "singlecast"
4643 video_encoder_config.simulcast_layers[0].active = false;
4644 video_encoder_config.simulcast_layers[1].active = false;
4645 video_encoder_config.simulcast_layers[2].active = true;
4646
4647 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
4648 kMaxPayloadLength);
4649 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4650
4651 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4652 // Frame should be dropped because of initial frame drop.
4653 ExpectDroppedFrame();
4654
4655 // Expect the sink_wants to specify a scaled frame.
4656 EXPECT_TRUE_WAIT(
4657 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
4658 video_stream_encoder_->Stop();
4659}
4660
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01004661TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
4662 const int kLowTargetBitrateBps = 400000;
4663 // Set simulcast.
4664 ResetEncoder("VP9", 1, 1, 3, false);
4665 fake_encoder_.SetQualityScaling(true);
4666 const int kWidth = 1280;
4667 const int kHeight = 720;
4668 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4669 DataRate::BitsPerSec(kLowTargetBitrateBps),
4670 DataRate::BitsPerSec(kLowTargetBitrateBps),
4671 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4672 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4673 // Frame should not be dropped.
4674 WaitForEncodedFrame(1);
4675
4676 // Trigger QVGA "singlecast"
4677 // Update the config.
4678 VideoEncoderConfig video_encoder_config;
4679 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
4680 &video_encoder_config);
4681 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4682 vp9_settings.numberOfSpatialLayers = 3;
4683 // Since only one layer is active - automatic resize should be enabled.
4684 vp9_settings.automaticResizeOn = true;
4685 video_encoder_config.encoder_specific_settings =
4686 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4687 vp9_settings);
4688 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
4689 video_encoder_config.content_type =
4690 VideoEncoderConfig::ContentType::kRealtimeVideo;
4691 // Currently simulcast layers |active| flags are used to inidicate
4692 // which SVC layers are active.
4693 video_encoder_config.simulcast_layers.resize(3);
4694
4695 video_encoder_config.simulcast_layers[0].active = true;
4696 video_encoder_config.simulcast_layers[1].active = false;
4697 video_encoder_config.simulcast_layers[2].active = false;
4698
4699 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
4700 kMaxPayloadLength);
4701 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4702
4703 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4704 // Frame should not be dropped.
4705 WaitForEncodedFrame(2);
4706
4707 // Trigger HD "singlecast"
4708 video_encoder_config.simulcast_layers[0].active = false;
4709 video_encoder_config.simulcast_layers[1].active = false;
4710 video_encoder_config.simulcast_layers[2].active = true;
4711
4712 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
4713 kMaxPayloadLength);
4714 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4715
4716 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4717 // Frame should be dropped because of initial frame drop.
4718 ExpectDroppedFrame();
4719
4720 // Expect the sink_wants to specify a scaled frame.
4721 EXPECT_TRUE_WAIT(
4722 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
4723 video_stream_encoder_->Stop();
4724}
4725
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01004726TEST_F(VideoStreamEncoderTest,
4727 InitialFrameDropActivatesWhenResolutionIncreases) {
4728 const int kWidth = 640;
4729 const int kHeight = 360;
4730
4731 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4732 DataRate::BitsPerSec(kTargetBitrateBps),
4733 DataRate::BitsPerSec(kTargetBitrateBps),
4734 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4735 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
4736 // Frame should not be dropped.
4737 WaitForEncodedFrame(1);
4738
4739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4740 DataRate::BitsPerSec(kLowTargetBitrateBps),
4741 DataRate::BitsPerSec(kLowTargetBitrateBps),
4742 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4743 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
4744 // Frame should not be dropped, bitrate not too low for frame.
4745 WaitForEncodedFrame(2);
4746
4747 // Incoming resolution increases.
4748 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4749 // Expect to drop this frame, bitrate too low for frame.
4750 ExpectDroppedFrame();
4751
4752 // Expect the sink_wants to specify a scaled frame.
4753 EXPECT_TRUE_WAIT(
4754 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
4755 video_stream_encoder_->Stop();
4756}
4757
4758TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
4759 const int kWidth = 640;
4760 const int kHeight = 360;
4761 // So that quality scaling doesn't happen by itself.
4762 fake_encoder_.SetQp(kQpHigh);
4763
4764 AdaptingFrameForwarder source(&time_controller_);
4765 source.set_adaptation_enabled(true);
4766 video_stream_encoder_->SetSource(
4767 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4768
4769 int timestamp = 1;
4770
4771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4772 DataRate::BitsPerSec(kTargetBitrateBps),
4773 DataRate::BitsPerSec(kTargetBitrateBps),
4774 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4775 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
4776 WaitForEncodedFrame(timestamp);
4777 timestamp += 9000;
4778 // Long pause to disable all first BWE drop logic.
4779 AdvanceTime(TimeDelta::Millis(1000));
4780
4781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4782 DataRate::BitsPerSec(kLowTargetBitrateBps),
4783 DataRate::BitsPerSec(kLowTargetBitrateBps),
4784 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4785 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
4786 // Not dropped frame, as initial frame drop is disabled by now.
4787 WaitForEncodedFrame(timestamp);
4788 timestamp += 9000;
4789 AdvanceTime(TimeDelta::Millis(100));
4790
4791 // Quality adaptation down.
4792 video_stream_encoder_->TriggerQualityLow();
4793
4794 // Adaptation has an effect.
4795 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4796 5000);
4797
4798 // Frame isn't dropped as initial frame dropper is disabled.
4799 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
4800 WaitForEncodedFrame(timestamp);
4801 timestamp += 9000;
4802 AdvanceTime(TimeDelta::Millis(100));
4803
4804 // Quality adaptation up.
4805 video_stream_encoder_->TriggerQualityHigh();
4806
4807 // Adaptation has an effect.
4808 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
4809 5000);
4810
4811 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
4812 // Frame should not be dropped, as initial framedropper is off.
4813 WaitForEncodedFrame(timestamp);
4814
4815 video_stream_encoder_->Stop();
4816}
4817
Åsa Perssone644a032019-11-08 15:56:00 +01004818TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4819 webrtc::test::ScopedFieldTrials field_trials(
4820 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4821
4822 // Reset encoder for field trials to take effect.
4823 VideoEncoderConfig config = video_encoder_config_.Copy();
4824 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004825 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004826 ConfigureEncoder(std::move(config));
4827 fake_encoder_.SetQp(kQpLow);
4828
4829 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004830 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01004831 source.set_adaptation_enabled(true);
4832 video_stream_encoder_->SetSource(&source,
4833 DegradationPreference::MAINTAIN_FRAMERATE);
4834
4835 // Start at low bitrate.
4836 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4838 DataRate::BitsPerSec(kLowBitrateBps),
4839 DataRate::BitsPerSec(kLowBitrateBps),
4840 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004841
4842 // Expect first frame to be dropped and resolution to be limited.
4843 const int kWidth = 1280;
4844 const int kHeight = 720;
4845 const int64_t kFrameIntervalMs = 100;
4846 int64_t timestamp_ms = kFrameIntervalMs;
4847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4848 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004849 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4850 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004851
4852 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4854 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004855
4856 // Insert frames and advance |min_duration_ms|.
4857 for (size_t i = 1; i <= 10; i++) {
4858 timestamp_ms += kFrameIntervalMs;
4859 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4860 WaitForEncodedFrame(timestamp_ms);
4861 }
4862 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4863 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4864
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004865 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004866
4867 // Insert frame should trigger high BW and release quality limitation.
4868 timestamp_ms += kFrameIntervalMs;
4869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4870 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004871 // The ramp-up code involves the adaptation queue, give it time to execute.
4872 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02004873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004874 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004875
4876 // Frame should not be adapted.
4877 timestamp_ms += kFrameIntervalMs;
4878 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4879 WaitForEncodedFrame(kWidth, kHeight);
4880 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4881
4882 video_stream_encoder_->Stop();
4883}
4884
mflodmancc3d4422017-08-03 08:27:51 -07004885TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004886 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004887 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004888 source.set_adaptation_enabled(true);
4889 video_stream_encoder_->SetSource(&source,
4890 DegradationPreference::MAINTAIN_FRAMERATE);
4891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4892 DataRate::BitsPerSec(kTargetBitrateBps),
4893 DataRate::BitsPerSec(kTargetBitrateBps),
4894 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4895 fake_encoder_.SetQp(kQpHigh + 1);
4896 const int kWidth = 1280;
4897 const int kHeight = 720;
4898 const int64_t kFrameIntervalMs = 100;
4899 int64_t timestamp_ms = kFrameIntervalMs;
4900 for (size_t i = 1; i <= 100; i++) {
4901 timestamp_ms += kFrameIntervalMs;
4902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4903 WaitForEncodedFrame(timestamp_ms);
4904 }
4905 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
4906 // for the first time.
4907 // TODO(eshr): We should avoid these waits by using threads with simulated
4908 // time.
4909 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
4910 2000 * 2.5 * 2);
4911 timestamp_ms += kFrameIntervalMs;
4912 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4913 WaitForEncodedFrame(timestamp_ms);
4914 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4915 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
4916 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4917
4918 // Disable Quality scaling by turning off scaler on the encoder and
4919 // reconfiguring.
4920 fake_encoder_.SetQualityScaling(false);
4921 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
4922 kMaxPayloadLength);
4923 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004924 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004925 // Since we turned off the quality scaler, the adaptations made by it are
4926 // removed.
4927 EXPECT_THAT(source.sink_wants(), ResolutionMax());
4928 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4929
4930 video_stream_encoder_->Stop();
4931}
4932
4933TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004934 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4935 const int kTooSmallWidth = 10;
4936 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004938 DataRate::BitsPerSec(kTargetBitrateBps),
4939 DataRate::BitsPerSec(kTargetBitrateBps),
4940 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004941
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004942 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004943 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004944 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004945 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004946 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004947 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4948
4949 // Trigger adapt down, too small frame, expect no change.
4950 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004951 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004952 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004953 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004954 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4955 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4956
mflodmancc3d4422017-08-03 08:27:51 -07004957 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004958}
4959
mflodmancc3d4422017-08-03 08:27:51 -07004960TEST_F(VideoStreamEncoderTest,
4961 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004962 const int kTooSmallWidth = 10;
4963 const int kTooSmallHeight = 10;
4964 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004965 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004966 DataRate::BitsPerSec(kTargetBitrateBps),
4967 DataRate::BitsPerSec(kTargetBitrateBps),
4968 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004969
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004970 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004971 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004972 video_stream_encoder_->SetSource(&source,
4973 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004974 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004975 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4976 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4977
4978 // Trigger adapt down, expect limited framerate.
4979 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004980 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004981 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004982 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004983 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4984 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4985 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4986
4987 // Trigger adapt down, too small frame, expect no change.
4988 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004989 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004990 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004991 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004992 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4993 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4994 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4995
mflodmancc3d4422017-08-03 08:27:51 -07004996 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004997}
4998
mflodmancc3d4422017-08-03 08:27:51 -07004999TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07005000 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02005001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005002 DataRate::BitsPerSec(kTargetBitrateBps),
5003 DataRate::BitsPerSec(kTargetBitrateBps),
5004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02005005 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07005006 const int kFrameWidth = 1280;
5007 const int kFrameHeight = 720;
5008 video_source_.IncomingCapturedFrame(
5009 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005010 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07005011 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07005012}
5013
sprangb1ca0732017-02-01 08:38:12 -08005014// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07005015TEST_F(VideoStreamEncoderTest,
5016 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02005017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005018 DataRate::BitsPerSec(kTargetBitrateBps),
5019 DataRate::BitsPerSec(kTargetBitrateBps),
5020 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08005021
5022 const int kFrameWidth = 1280;
5023 const int kFrameHeight = 720;
5024 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07005025 // requested by
5026 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08005027 video_source_.set_adaptation_enabled(true);
5028
5029 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005030 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005031 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005032
5033 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005034 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08005035 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005036 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005037 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08005038
asaperssonfab67072017-04-04 05:51:49 -07005039 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02005040 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08005041 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005042 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005043 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005044
mflodmancc3d4422017-08-03 08:27:51 -07005045 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08005046}
sprangfe627f32017-03-29 08:24:59 -07005047
mflodmancc3d4422017-08-03 08:27:51 -07005048TEST_F(VideoStreamEncoderTest,
5049 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07005050 const int kFrameWidth = 1280;
5051 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07005052
Henrik Boström381d1092020-05-12 18:49:07 +02005053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005054 DataRate::BitsPerSec(kTargetBitrateBps),
5055 DataRate::BitsPerSec(kTargetBitrateBps),
5056 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005057 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005058 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005059 video_source_.set_adaptation_enabled(true);
5060
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005061 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005062
5063 video_source_.IncomingCapturedFrame(
5064 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005065 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005066
5067 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07005068 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005069
5070 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07005071 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005072 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005073 video_source_.IncomingCapturedFrame(
5074 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005075 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005076 }
5077
5078 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07005079 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005080 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005081 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005082 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005083 video_source_.IncomingCapturedFrame(
5084 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005085 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005086 ++num_frames_dropped;
5087 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005088 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005089 }
5090 }
5091
sprang4847ae62017-06-27 07:06:52 -07005092 // Add some slack to account for frames dropped by the frame dropper.
5093 const int kErrorMargin = 1;
5094 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005095 kErrorMargin);
5096
5097 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07005098 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005099 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02005100 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005101 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005102 video_source_.IncomingCapturedFrame(
5103 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005104 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005105 ++num_frames_dropped;
5106 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005107 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005108 }
5109 }
sprang4847ae62017-06-27 07:06:52 -07005110 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07005111 kErrorMargin);
5112
5113 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02005114 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005115 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005116 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005117 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005118 video_source_.IncomingCapturedFrame(
5119 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005120 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005121 ++num_frames_dropped;
5122 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005123 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005124 }
5125 }
sprang4847ae62017-06-27 07:06:52 -07005126 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005127 kErrorMargin);
5128
5129 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02005130 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005131 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005132 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005133 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005134 video_source_.IncomingCapturedFrame(
5135 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005136 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005137 ++num_frames_dropped;
5138 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005139 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005140 }
5141 }
5142 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
5143
mflodmancc3d4422017-08-03 08:27:51 -07005144 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005145}
5146
mflodmancc3d4422017-08-03 08:27:51 -07005147TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07005148 const int kFramerateFps = 5;
5149 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07005150 const int kFrameWidth = 1280;
5151 const int kFrameHeight = 720;
5152
sprang4847ae62017-06-27 07:06:52 -07005153 // Reconfigure encoder with two temporal layers and screensharing, which will
5154 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02005155 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07005156
Henrik Boström381d1092020-05-12 18:49:07 +02005157 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005158 DataRate::BitsPerSec(kTargetBitrateBps),
5159 DataRate::BitsPerSec(kTargetBitrateBps),
5160 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005161 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005162 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005163 video_source_.set_adaptation_enabled(true);
5164
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005165 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005166
5167 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08005168 rtc::VideoSinkWants last_wants;
5169 do {
5170 last_wants = video_source_.sink_wants();
5171
sprangc5d62e22017-04-02 23:53:04 -07005172 // Insert frames to get a new fps estimate...
5173 for (int j = 0; j < kFramerateFps; ++j) {
5174 video_source_.IncomingCapturedFrame(
5175 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08005176 if (video_source_.last_sent_width()) {
5177 sink_.WaitForEncodedFrame(timestamp_ms);
5178 }
sprangc5d62e22017-04-02 23:53:04 -07005179 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005180 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07005181 }
5182 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07005183 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08005184 } while (video_source_.sink_wants().max_framerate_fps <
5185 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07005186
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005187 EXPECT_THAT(video_source_.sink_wants(),
5188 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07005189
mflodmancc3d4422017-08-03 08:27:51 -07005190 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005191}
asaperssonf7e294d2017-06-13 23:25:22 -07005192
mflodmancc3d4422017-08-03 08:27:51 -07005193TEST_F(VideoStreamEncoderTest,
5194 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005195 const int kWidth = 1280;
5196 const int kHeight = 720;
5197 const int64_t kFrameIntervalMs = 150;
5198 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005199 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005200 DataRate::BitsPerSec(kTargetBitrateBps),
5201 DataRate::BitsPerSec(kTargetBitrateBps),
5202 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005203
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005204 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005205 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005206 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005207 video_stream_encoder_->SetSource(&source,
5208 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005209 timestamp_ms += kFrameIntervalMs;
5210 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005211 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005212 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005213 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5214 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5215 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5216
5217 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005218 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005219 timestamp_ms += kFrameIntervalMs;
5220 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005221 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005222 EXPECT_THAT(source.sink_wants(),
5223 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005224 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5225 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5226 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5227
5228 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005229 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005230 timestamp_ms += kFrameIntervalMs;
5231 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005232 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005233 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005234 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5235 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5236 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5237
5238 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005239 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005240 timestamp_ms += kFrameIntervalMs;
5241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005242 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005243 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005244 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5245 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5246 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5247
5248 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005249 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005250 timestamp_ms += kFrameIntervalMs;
5251 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005252 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005253 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005254 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5255 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5256 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5257
5258 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005259 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005260 timestamp_ms += kFrameIntervalMs;
5261 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005262 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005263 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005264 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5265 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5266 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5267
5268 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005269 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005270 timestamp_ms += kFrameIntervalMs;
5271 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005272 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005273 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005274 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5275 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5276 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5277
5278 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07005279 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005280 timestamp_ms += kFrameIntervalMs;
5281 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005282 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005283 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005284 rtc::VideoSinkWants last_wants = source.sink_wants();
5285 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5287 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5288
5289 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005290 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005291 timestamp_ms += kFrameIntervalMs;
5292 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005293 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005294 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07005295 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5296 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5297 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5298
Evan Shrubsole64469032020-06-11 10:45:29 +02005299 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005300 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005301 timestamp_ms += kFrameIntervalMs;
5302 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005303 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005304 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005305 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5306 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5307 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5308
5309 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005310 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005311 timestamp_ms += kFrameIntervalMs;
5312 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005313 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005314 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5317 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5318
5319 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005320 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005321 timestamp_ms += kFrameIntervalMs;
5322 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005323 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005324 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005325 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5326 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5327 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5328
5329 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005330 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005331 timestamp_ms += kFrameIntervalMs;
5332 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005333 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005334 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005335 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5336 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5337 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5338
5339 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005340 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005341 timestamp_ms += kFrameIntervalMs;
5342 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005343 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005344 EXPECT_THAT(source.sink_wants(), FpsMax());
5345 EXPECT_EQ(source.sink_wants().max_pixel_count,
5346 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07005347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5348 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5349 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5350
5351 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005352 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005353 timestamp_ms += kFrameIntervalMs;
5354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005355 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005356 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5358 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5359 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5360
Åsa Persson30ab0152019-08-27 12:22:33 +02005361 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005362 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005363 timestamp_ms += kFrameIntervalMs;
5364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005365 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005366 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005367 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005368 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5369 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5370 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5371
5372 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005373 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005374 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005375 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5376
mflodmancc3d4422017-08-03 08:27:51 -07005377 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005378}
5379
mflodmancc3d4422017-08-03 08:27:51 -07005380TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07005381 const int kWidth = 1280;
5382 const int kHeight = 720;
5383 const int64_t kFrameIntervalMs = 150;
5384 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005385 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005386 DataRate::BitsPerSec(kTargetBitrateBps),
5387 DataRate::BitsPerSec(kTargetBitrateBps),
5388 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005389
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005390 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005391 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005392 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005393 video_stream_encoder_->SetSource(&source,
5394 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005395 timestamp_ms += kFrameIntervalMs;
5396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005397 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005398 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005399 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5401 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5402 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5403 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5404 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5405
5406 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005407 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005408 timestamp_ms += kFrameIntervalMs;
5409 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005410 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005411 EXPECT_THAT(source.sink_wants(),
5412 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5415 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5416 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5417 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5418 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5419
5420 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005421 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005422 timestamp_ms += kFrameIntervalMs;
5423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005424 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005425 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5428 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5429 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5430 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5431 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5432
5433 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005434 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005435 timestamp_ms += kFrameIntervalMs;
5436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005437 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005438 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02005439 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07005440 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5441 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5443 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5444 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5445
Evan Shrubsole64469032020-06-11 10:45:29 +02005446 // Trigger cpu adapt up, expect no change since QP is most limited.
5447 {
5448 // Store current sink wants since we expect no change and if there is no
5449 // change then last_wants() is not updated.
5450 auto previous_sink_wants = source.sink_wants();
5451 video_stream_encoder_->TriggerCpuUnderuse();
5452 timestamp_ms += kFrameIntervalMs;
5453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5454 WaitForEncodedFrame(timestamp_ms);
5455 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
5456 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5457 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5458 }
5459
5460 // Trigger quality adapt up, expect increased fps (640x360@30fps).
5461 video_stream_encoder_->TriggerQualityHigh();
5462 timestamp_ms += kFrameIntervalMs;
5463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5464 WaitForEncodedFrame(timestamp_ms);
5465 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
5466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5469 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5470 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5471 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5472
5473 // Trigger quality adapt up and Cpu adapt up since both are most limited,
5474 // expect increased resolution (960x540@30fps).
5475 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02005476 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005477 timestamp_ms += kFrameIntervalMs;
5478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005479 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02005480 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5482 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5483 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5484 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5485 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005486 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005487
Evan Shrubsole64469032020-06-11 10:45:29 +02005488 // Trigger quality adapt up and Cpu adapt up since both are most limited,
5489 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005490 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02005491 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005492 timestamp_ms += kFrameIntervalMs;
5493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005494 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005495 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005496 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5499 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5500 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5501 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005502 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005503
5504 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005505 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005506 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005507 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005508 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005509
mflodmancc3d4422017-08-03 08:27:51 -07005510 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005511}
5512
mflodmancc3d4422017-08-03 08:27:51 -07005513TEST_F(VideoStreamEncoderTest,
5514 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07005515 const int kWidth = 640;
5516 const int kHeight = 360;
5517 const int kFpsLimit = 15;
5518 const int64_t kFrameIntervalMs = 150;
5519 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005520 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005521 DataRate::BitsPerSec(kTargetBitrateBps),
5522 DataRate::BitsPerSec(kTargetBitrateBps),
5523 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005524
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005525 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005526 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005527 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005528 video_stream_encoder_->SetSource(&source,
5529 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005530 timestamp_ms += kFrameIntervalMs;
5531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005532 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005533 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5536 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5537 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5538 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5539 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5540
5541 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005542 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005543 timestamp_ms += kFrameIntervalMs;
5544 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005545 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005546 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5549 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5550 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5551 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5552 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5553
5554 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005555 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005556 timestamp_ms += kFrameIntervalMs;
5557 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005558 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005559 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02005561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07005562 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5563 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5564 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5565 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5566
Evan Shrubsole64469032020-06-11 10:45:29 +02005567 // Trigger cpu adapt up, expect no change because quality is most limited.
5568 {
5569 auto previous_sink_wants = source.sink_wants();
5570 // Store current sink wants since we expect no change ind if there is no
5571 // change then last__wants() is not updated.
5572 video_stream_encoder_->TriggerCpuUnderuse();
5573 timestamp_ms += kFrameIntervalMs;
5574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5575 WaitForEncodedFrame(timestamp_ms);
5576 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
5577 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5578 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5579 }
5580
5581 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
5582 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005583 timestamp_ms += kFrameIntervalMs;
5584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005585 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005586 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005587 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5589 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02005590 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5591 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5592 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005593
Evan Shrubsole64469032020-06-11 10:45:29 +02005594 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005595 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02005596 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005597 timestamp_ms += kFrameIntervalMs;
5598 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005599 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005600 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5602 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5603 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5604 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5605 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005606 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005607
5608 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005609 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005610 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005611 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005612 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005613
mflodmancc3d4422017-08-03 08:27:51 -07005614 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005615}
5616
mflodmancc3d4422017-08-03 08:27:51 -07005617TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07005618 const int kFrameWidth = 1920;
5619 const int kFrameHeight = 1080;
5620 // 3/4 of 1920.
5621 const int kAdaptedFrameWidth = 1440;
5622 // 3/4 of 1080 rounded down to multiple of 4.
5623 const int kAdaptedFrameHeight = 808;
5624 const int kFramerate = 24;
5625
Henrik Boström381d1092020-05-12 18:49:07 +02005626 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005627 DataRate::BitsPerSec(kTargetBitrateBps),
5628 DataRate::BitsPerSec(kTargetBitrateBps),
5629 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07005630 // Trigger reconfigure encoder (without resetting the entire instance).
5631 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02005632 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5633 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07005634 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07005635 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02005636 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07005637 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005638 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005639 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07005640
5641 video_source_.set_adaptation_enabled(true);
5642
5643 video_source_.IncomingCapturedFrame(
5644 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005645 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005646
5647 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005648 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07005649 video_source_.IncomingCapturedFrame(
5650 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005651 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005652
mflodmancc3d4422017-08-03 08:27:51 -07005653 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07005654}
5655
mflodmancc3d4422017-08-03 08:27:51 -07005656TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07005657 const int kFrameWidth = 1280;
5658 const int kFrameHeight = 720;
5659 const int kLowFps = 2;
5660 const int kHighFps = 30;
5661
Henrik Boström381d1092020-05-12 18:49:07 +02005662 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005663 DataRate::BitsPerSec(kTargetBitrateBps),
5664 DataRate::BitsPerSec(kTargetBitrateBps),
5665 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005666
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005667 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07005668 max_framerate_ = kLowFps;
5669
5670 // Insert 2 seconds of 2fps video.
5671 for (int i = 0; i < kLowFps * 2; ++i) {
5672 video_source_.IncomingCapturedFrame(
5673 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5674 WaitForEncodedFrame(timestamp_ms);
5675 timestamp_ms += 1000 / kLowFps;
5676 }
5677
5678 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02005679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005680 DataRate::BitsPerSec(kTargetBitrateBps),
5681 DataRate::BitsPerSec(kTargetBitrateBps),
5682 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005683 video_source_.IncomingCapturedFrame(
5684 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5685 WaitForEncodedFrame(timestamp_ms);
5686 timestamp_ms += 1000 / kLowFps;
5687
5688 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
5689
5690 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02005691 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07005692 const int kFrameIntervalMs = 1000 / kHighFps;
5693 max_framerate_ = kHighFps;
5694 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
5695 video_source_.IncomingCapturedFrame(
5696 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5697 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
5698 // be dropped if the encoder hans't been updated with the new higher target
5699 // framerate yet, causing it to overshoot the target bitrate and then
5700 // suffering the wrath of the media optimizer.
5701 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
5702 timestamp_ms += kFrameIntervalMs;
5703 }
5704
5705 // Don expect correct measurement just yet, but it should be higher than
5706 // before.
5707 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
5708
mflodmancc3d4422017-08-03 08:27:51 -07005709 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005710}
5711
mflodmancc3d4422017-08-03 08:27:51 -07005712TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07005713 const int kFrameWidth = 1280;
5714 const int kFrameHeight = 720;
5715 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005716 ResetEncoder("FAKE", 1, 1, 1, false,
5717 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
5718 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07005719
Henrik Boström381d1092020-05-12 18:49:07 +02005720 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005721 DataRate::BitsPerSec(kTargetBitrateBps),
5722 DataRate::BitsPerSec(kTargetBitrateBps),
5723 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005724 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005725
5726 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005727 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07005728 video_source_.IncomingCapturedFrame(
5729 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5730 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02005731 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07005732
5733 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005734 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5735 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5736 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005737
5738 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005739 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005740 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005741
Per Kjellanderdcef6412020-10-07 15:09:05 +02005742 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07005743 video_source_.IncomingCapturedFrame(
5744 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5745 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02005746 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07005747
mflodmancc3d4422017-08-03 08:27:51 -07005748 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005749}
ilnik6b826ef2017-06-16 06:53:48 -07005750
Niels Möller4db138e2018-04-19 09:04:13 +02005751TEST_F(VideoStreamEncoderTest,
5752 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5753 const int kFrameWidth = 1280;
5754 const int kFrameHeight = 720;
5755 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005757 DataRate::BitsPerSec(kTargetBitrateBps),
5758 DataRate::BitsPerSec(kTargetBitrateBps),
5759 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005760 video_source_.IncomingCapturedFrame(
5761 CreateFrame(1, kFrameWidth, kFrameHeight));
5762 WaitForEncodedFrame(1);
5763 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5764 .low_encode_usage_threshold_percent,
5765 default_options.low_encode_usage_threshold_percent);
5766 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5767 .high_encode_usage_threshold_percent,
5768 default_options.high_encode_usage_threshold_percent);
5769 video_stream_encoder_->Stop();
5770}
5771
5772TEST_F(VideoStreamEncoderTest,
5773 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5774 const int kFrameWidth = 1280;
5775 const int kFrameHeight = 720;
5776 CpuOveruseOptions hardware_options;
5777 hardware_options.low_encode_usage_threshold_percent = 150;
5778 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005779 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005780
Henrik Boström381d1092020-05-12 18:49:07 +02005781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005782 DataRate::BitsPerSec(kTargetBitrateBps),
5783 DataRate::BitsPerSec(kTargetBitrateBps),
5784 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005785 video_source_.IncomingCapturedFrame(
5786 CreateFrame(1, kFrameWidth, kFrameHeight));
5787 WaitForEncodedFrame(1);
5788 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5789 .low_encode_usage_threshold_percent,
5790 hardware_options.low_encode_usage_threshold_percent);
5791 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5792 .high_encode_usage_threshold_percent,
5793 hardware_options.high_encode_usage_threshold_percent);
5794 video_stream_encoder_->Stop();
5795}
5796
Niels Möller6bb5ab92019-01-11 11:11:10 +01005797TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5798 const int kFrameWidth = 320;
5799 const int kFrameHeight = 240;
5800 const int kFps = 30;
5801 const int kTargetBitrateBps = 120000;
5802 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5803
Henrik Boström381d1092020-05-12 18:49:07 +02005804 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005805 DataRate::BitsPerSec(kTargetBitrateBps),
5806 DataRate::BitsPerSec(kTargetBitrateBps),
5807 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005808
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005809 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01005810 max_framerate_ = kFps;
5811
5812 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5813 fake_encoder_.SimulateOvershoot(1.0);
5814 int num_dropped = 0;
5815 for (int i = 0; i < kNumFramesInRun; ++i) {
5816 video_source_.IncomingCapturedFrame(
5817 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5818 // Wait up to two frame durations for a frame to arrive.
5819 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5820 ++num_dropped;
5821 }
5822 timestamp_ms += 1000 / kFps;
5823 }
5824
Erik Språnga8d48ab2019-02-08 14:17:40 +01005825 // Framerate should be measured to be near the expected target rate.
5826 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5827
5828 // Frame drops should be within 5% of expected 0%.
5829 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005830
5831 // Make encoder produce frames at double the expected bitrate during 3 seconds
5832 // of video, verify number of drops. Rate needs to be slightly changed in
5833 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005834 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02005835 const RateControlSettings trials =
5836 RateControlSettings::ParseFromFieldTrials();
5837 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01005838 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02005839 // frame dropping since the adjuter will try to just lower the target
5840 // bitrate rather than drop frames. If network headroom can be used, it
5841 // doesn't push back as hard so we don't need quite as much overshoot.
5842 // These numbers are unfortunately a bit magical but there's not trivial
5843 // way to algebraically infer them.
5844 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
5845 overshoot_factor = 2.4;
5846 } else {
5847 overshoot_factor = 4.0;
5848 }
Erik Språng7ca375c2019-02-06 16:20:17 +01005849 }
5850 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005852 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5853 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5854 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005855 num_dropped = 0;
5856 for (int i = 0; i < kNumFramesInRun; ++i) {
5857 video_source_.IncomingCapturedFrame(
5858 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5859 // Wait up to two frame durations for a frame to arrive.
5860 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5861 ++num_dropped;
5862 }
5863 timestamp_ms += 1000 / kFps;
5864 }
5865
Henrik Boström381d1092020-05-12 18:49:07 +02005866 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005867 DataRate::BitsPerSec(kTargetBitrateBps),
5868 DataRate::BitsPerSec(kTargetBitrateBps),
5869 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005870
5871 // Target framerate should be still be near the expected target, despite
5872 // the frame drops.
5873 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5874
5875 // Frame drops should be within 5% of expected 50%.
5876 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005877
5878 video_stream_encoder_->Stop();
5879}
5880
5881TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5882 const int kFrameWidth = 320;
5883 const int kFrameHeight = 240;
5884 const int kActualInputFps = 24;
5885 const int kTargetBitrateBps = 120000;
5886
5887 ASSERT_GT(max_framerate_, kActualInputFps);
5888
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005889 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01005890 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005892 DataRate::BitsPerSec(kTargetBitrateBps),
5893 DataRate::BitsPerSec(kTargetBitrateBps),
5894 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005895
5896 // Insert 3 seconds of video, with an input fps lower than configured max.
5897 for (int i = 0; i < kActualInputFps * 3; ++i) {
5898 video_source_.IncomingCapturedFrame(
5899 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5900 // Wait up to two frame durations for a frame to arrive.
5901 WaitForEncodedFrame(timestamp_ms);
5902 timestamp_ms += 1000 / kActualInputFps;
5903 }
5904
5905 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5906
5907 video_stream_encoder_->Stop();
5908}
5909
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005910TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005911 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005912 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005913 DataRate::BitsPerSec(kTargetBitrateBps),
5914 DataRate::BitsPerSec(kTargetBitrateBps),
5915 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005916
5917 fake_encoder_.BlockNextEncode();
5918 video_source_.IncomingCapturedFrame(
5919 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5920 WaitForEncodedFrame(1);
5921 // On the very first frame full update should be forced.
5922 rect = fake_encoder_.GetLastUpdateRect();
5923 EXPECT_EQ(rect.offset_x, 0);
5924 EXPECT_EQ(rect.offset_y, 0);
5925 EXPECT_EQ(rect.height, codec_height_);
5926 EXPECT_EQ(rect.width, codec_width_);
5927 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5928 // call to ContinueEncode.
5929 video_source_.IncomingCapturedFrame(
5930 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5931 ExpectDroppedFrame();
5932 video_source_.IncomingCapturedFrame(
5933 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5934 ExpectDroppedFrame();
5935 fake_encoder_.ContinueEncode();
5936 WaitForEncodedFrame(3);
5937 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5938 rect = fake_encoder_.GetLastUpdateRect();
5939 EXPECT_EQ(rect.offset_x, 1);
5940 EXPECT_EQ(rect.offset_y, 0);
5941 EXPECT_EQ(rect.width, 10);
5942 EXPECT_EQ(rect.height, 1);
5943
5944 video_source_.IncomingCapturedFrame(
5945 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5946 WaitForEncodedFrame(4);
5947 // Previous frame was encoded, so no accumulation should happen.
5948 rect = fake_encoder_.GetLastUpdateRect();
5949 EXPECT_EQ(rect.offset_x, 0);
5950 EXPECT_EQ(rect.offset_y, 0);
5951 EXPECT_EQ(rect.width, 1);
5952 EXPECT_EQ(rect.height, 1);
5953
5954 video_stream_encoder_->Stop();
5955}
5956
Erik Språngd7329ca2019-02-21 21:19:53 +01005957TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005959 DataRate::BitsPerSec(kTargetBitrateBps),
5960 DataRate::BitsPerSec(kTargetBitrateBps),
5961 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005962
5963 // First frame is always keyframe.
5964 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5965 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005966 EXPECT_THAT(
5967 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005968 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005969
5970 // Insert delta frame.
5971 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5972 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005973 EXPECT_THAT(
5974 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005975 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005976
5977 // Request next frame be a key-frame.
5978 video_stream_encoder_->SendKeyFrame();
5979 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5980 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005981 EXPECT_THAT(
5982 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005983 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005984
5985 video_stream_encoder_->Stop();
5986}
5987
5988TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5989 // Setup simulcast with three streams.
5990 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005992 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5993 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5994 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005995 // Wait for all three layers before triggering event.
5996 sink_.SetNumExpectedLayers(3);
5997
5998 // First frame is always keyframe.
5999 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6000 WaitForEncodedFrame(1);
6001 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006002 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6003 VideoFrameType::kVideoFrameKey,
6004 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006005
6006 // Insert delta frame.
6007 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6008 WaitForEncodedFrame(2);
6009 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006010 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6011 VideoFrameType::kVideoFrameDelta,
6012 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006013
6014 // Request next frame be a key-frame.
6015 // Only first stream is configured to produce key-frame.
6016 video_stream_encoder_->SendKeyFrame();
6017 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6018 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02006019
6020 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
6021 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01006022 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006023 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02006024 VideoFrameType::kVideoFrameKey,
6025 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006026
6027 video_stream_encoder_->Stop();
6028}
6029
6030TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
6031 // Configure internal source factory and setup test again.
6032 encoder_factory_.SetHasInternalSource(true);
6033 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006035 DataRate::BitsPerSec(kTargetBitrateBps),
6036 DataRate::BitsPerSec(kTargetBitrateBps),
6037 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006038
6039 // Call encoder directly, simulating internal source where encoded frame
6040 // callback in VideoStreamEncoder is called despite no OnFrame().
6041 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
6042 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006043 EXPECT_THAT(
6044 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006045 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006046
Niels Möller8f7ce222019-03-21 15:43:58 +01006047 const std::vector<VideoFrameType> kDeltaFrame = {
6048 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01006049 // Need to set timestamp manually since manually for injected frame.
6050 VideoFrame frame = CreateFrame(101, nullptr);
6051 frame.set_timestamp(101);
6052 fake_encoder_.InjectFrame(frame, false);
6053 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006054 EXPECT_THAT(
6055 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006056 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006057
6058 // Request key-frame. The forces a dummy frame down into the encoder.
6059 fake_encoder_.ExpectNullFrame();
6060 video_stream_encoder_->SendKeyFrame();
6061 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006062 EXPECT_THAT(
6063 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006064 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006065
6066 video_stream_encoder_->Stop();
6067}
Erik Språngb7cb7b52019-02-26 15:52:33 +01006068
6069TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
6070 // Configure internal source factory and setup test again.
6071 encoder_factory_.SetHasInternalSource(true);
6072 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006074 DataRate::BitsPerSec(kTargetBitrateBps),
6075 DataRate::BitsPerSec(kTargetBitrateBps),
6076 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006077
6078 int64_t timestamp = 1;
6079 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02006080 image.SetEncodedData(
6081 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01006082 image.capture_time_ms_ = ++timestamp;
6083 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
6084 const int64_t kEncodeFinishDelayMs = 10;
6085 image.timing_.encode_start_ms = timestamp;
6086 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
6087 fake_encoder_.InjectEncodedImage(image);
6088 // Wait for frame without incrementing clock.
6089 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
6090 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
6091 // capture timestamp should be kEncodeFinishDelayMs in the past.
6092 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006093 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006094
6095 video_stream_encoder_->Stop();
6096}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006097
6098TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006099 // SPS contains VUI with restrictions on the maximum number of reordered
6100 // pictures, there is no need to rewrite the bitstream to enable faster
6101 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006102 ResetEncoder("H264", 1, 1, 1, false);
6103
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006104 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6105 DataRate::BitsPerSec(kTargetBitrateBps),
6106 DataRate::BitsPerSec(kTargetBitrateBps),
6107 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6108 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006109
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006110 fake_encoder_.SetEncodedImageData(
6111 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006112
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006113 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6114 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006115
6116 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6117 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006118
6119 video_stream_encoder_->Stop();
6120}
6121
6122TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006123 // SPS does not contain VUI, the bitstream is will be rewritten with added
6124 // VUI with restrictions on the maximum number of reordered pictures to
6125 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006126 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
6127 0x00, 0x00, 0x03, 0x03, 0xF4,
6128 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006129 ResetEncoder("H264", 1, 1, 1, false);
6130
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6132 DataRate::BitsPerSec(kTargetBitrateBps),
6133 DataRate::BitsPerSec(kTargetBitrateBps),
6134 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6135 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006136
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006137 fake_encoder_.SetEncodedImageData(
6138 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006139
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006140 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6141 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006142
6143 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6144 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006145
6146 video_stream_encoder_->Stop();
6147}
6148
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006149TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
6150 const int kFrameWidth = 1280;
6151 const int kFrameHeight = 720;
6152 const int kTargetBitrateBps = 300000; // To low for HD resolution.
6153
Henrik Boström381d1092020-05-12 18:49:07 +02006154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006155 DataRate::BitsPerSec(kTargetBitrateBps),
6156 DataRate::BitsPerSec(kTargetBitrateBps),
6157 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006158 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6159
6160 // Insert a first video frame. It should be dropped because of downscale in
6161 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006162 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006163 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6164 frame.set_rotation(kVideoRotation_270);
6165 video_source_.IncomingCapturedFrame(frame);
6166
6167 ExpectDroppedFrame();
6168
6169 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006170 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006171 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6172 frame.set_rotation(kVideoRotation_90);
6173 video_source_.IncomingCapturedFrame(frame);
6174
6175 WaitForEncodedFrame(timestamp_ms);
6176 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
6177
6178 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006179 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006180 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6181 frame.set_rotation(kVideoRotation_180);
6182 video_source_.IncomingCapturedFrame(frame);
6183
6184 WaitForEncodedFrame(timestamp_ms);
6185 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
6186
6187 video_stream_encoder_->Stop();
6188}
6189
Erik Språng5056af02019-09-02 15:53:11 +02006190TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
6191 const int kFrameWidth = 320;
6192 const int kFrameHeight = 180;
6193
6194 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02006195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006196 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
6197 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
6198 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02006199 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006200 /*rtt_ms=*/0,
6201 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006202
6203 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006204 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02006205 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6206 frame.set_rotation(kVideoRotation_270);
6207 video_source_.IncomingCapturedFrame(frame);
6208 WaitForEncodedFrame(timestamp_ms);
6209
6210 // Set a target rate below the minimum allowed by the codec settings.
6211 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006212 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
6213 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02006214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02006215 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02006216 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02006217 /*link_allocation=*/target_rate,
6218 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006219 /*rtt_ms=*/0,
6220 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006221 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6222
6223 // Target bitrate and bandwidth allocation should both be capped at min_rate.
6224 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6225 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006226 DataRate allocation_sum =
6227 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02006228 EXPECT_EQ(min_rate, allocation_sum);
6229 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
6230
6231 video_stream_encoder_->Stop();
6232}
6233
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006234TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02006235 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006236 DataRate::BitsPerSec(kTargetBitrateBps),
6237 DataRate::BitsPerSec(kTargetBitrateBps),
6238 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006239 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006240 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006241 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6242 WaitForEncodedFrame(1);
6243
6244 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6245 ASSERT_TRUE(prev_rate_settings.has_value());
6246 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
6247 kDefaultFramerate);
6248
6249 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
6250 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
6251 timestamp_ms += 1000 / kDefaultFramerate;
6252 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6253 WaitForEncodedFrame(timestamp_ms);
6254 }
6255 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
6256 kDefaultFramerate);
6257 // Capture larger frame to trigger a reconfigure.
6258 codec_height_ *= 2;
6259 codec_width_ *= 2;
6260 timestamp_ms += 1000 / kDefaultFramerate;
6261 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6262 WaitForEncodedFrame(timestamp_ms);
6263
6264 EXPECT_EQ(2, sink_.number_of_reconfigurations());
6265 auto current_rate_settings =
6266 fake_encoder_.GetAndResetLastRateControlSettings();
6267 // Ensure we have actually reconfigured twice
6268 // The rate settings should have been set again even though
6269 // they haven't changed.
6270 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006271 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006272
6273 video_stream_encoder_->Stop();
6274}
6275
philipeld9cc8c02019-09-16 14:53:40 +02006276struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02006277 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
6278 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
6279 MOCK_METHOD(void,
6280 RequestEncoderSwitch,
6281 (const webrtc::SdpVideoFormat& format),
6282 (override));
philipeld9cc8c02019-09-16 14:53:40 +02006283};
6284
6285TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
6286 constexpr int kDontCare = 100;
6287
6288 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6289 video_send_config_.encoder_settings.encoder_switch_request_callback =
6290 &switch_callback;
6291 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6292 encoder_config.codec_type = kVideoCodecVP8;
6293 webrtc::test::ScopedFieldTrials field_trial(
6294 "WebRTC-NetworkCondition-EncoderSwitch/"
6295 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6296 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6297
6298 // Reset encoder for new configuration to take effect.
6299 ConfigureEncoder(std::move(encoder_config));
6300
6301 // Send one frame to trigger ReconfigureEncoder.
6302 video_source_.IncomingCapturedFrame(
6303 CreateFrame(kDontCare, kDontCare, kDontCare));
6304
6305 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006306 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6307 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006308 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006309 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006310
Henrik Boström381d1092020-05-12 18:49:07 +02006311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006312 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6313 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6314 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02006315 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006316 /*rtt_ms=*/0,
6317 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006318 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02006319
6320 video_stream_encoder_->Stop();
6321}
6322
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006323TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
6324 constexpr int kDontCare = 100;
6325
6326 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6327 video_send_config_.encoder_settings.encoder_switch_request_callback =
6328 &switch_callback;
6329 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6330 encoder_config.codec_type = kVideoCodecVP8;
6331 webrtc::test::ScopedFieldTrials field_trial(
6332 "WebRTC-NetworkCondition-EncoderSwitch/"
6333 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6334 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6335
6336 // Reset encoder for new configuration to take effect.
6337 ConfigureEncoder(std::move(encoder_config));
6338
6339 // Send one frame to trigger ReconfigureEncoder.
6340 video_source_.IncomingCapturedFrame(
6341 CreateFrame(kDontCare, kDontCare, kDontCare));
6342
6343 using Config = EncoderSwitchRequestCallback::Config;
6344 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
6345 .Times(0);
6346
Henrik Boström381d1092020-05-12 18:49:07 +02006347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006348 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
6349 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
6350 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
6351 /*fraction_lost=*/0,
6352 /*rtt_ms=*/0,
6353 /*cwnd_reduce_ratio=*/0);
6354
6355 video_stream_encoder_->Stop();
6356}
6357
philipeld9cc8c02019-09-16 14:53:40 +02006358TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
6359 constexpr int kSufficientBitrateToNotDrop = 1000;
6360 constexpr int kHighRes = 500;
6361 constexpr int kLowRes = 100;
6362
6363 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6364 video_send_config_.encoder_settings.encoder_switch_request_callback =
6365 &switch_callback;
6366 webrtc::test::ScopedFieldTrials field_trial(
6367 "WebRTC-NetworkCondition-EncoderSwitch/"
6368 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
6369 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6370 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6371 encoder_config.codec_type = kVideoCodecH264;
6372
6373 // Reset encoder for new configuration to take effect.
6374 ConfigureEncoder(std::move(encoder_config));
6375
6376 // The VideoStreamEncoder needs some bitrate before it can start encoding,
6377 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
6378 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02006379 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006380 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6381 /*stable_target_bitrate=*/
6382 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6383 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02006384 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006385 /*rtt_ms=*/0,
6386 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02006387
6388 // Send one frame to trigger ReconfigureEncoder.
6389 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
6390 WaitForEncodedFrame(1);
6391
6392 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006393 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6394 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006395 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006396 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006397
6398 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
6399 WaitForEncodedFrame(2);
6400
6401 video_stream_encoder_->Stop();
6402}
6403
philipel9b058032020-02-10 11:30:00 +01006404TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
6405 constexpr int kDontCare = 100;
6406 StrictMock<MockEncoderSelector> encoder_selector;
6407 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6408 &fake_encoder_, &encoder_selector);
6409 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6410
6411 // Reset encoder for new configuration to take effect.
6412 ConfigureEncoder(video_encoder_config_.Copy());
6413
6414 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
6415
6416 video_source_.IncomingCapturedFrame(
6417 CreateFrame(kDontCare, kDontCare, kDontCare));
6418 video_stream_encoder_->Stop();
6419
6420 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
6421 // to it's factory, so in order for the encoder instance in the
6422 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
6423 // reset the |video_stream_encoder_| here.
6424 video_stream_encoder_.reset();
6425}
6426
6427TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
6428 constexpr int kDontCare = 100;
6429
6430 NiceMock<MockEncoderSelector> encoder_selector;
6431 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6432 video_send_config_.encoder_settings.encoder_switch_request_callback =
6433 &switch_callback;
6434 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6435 &fake_encoder_, &encoder_selector);
6436 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6437
6438 // Reset encoder for new configuration to take effect.
6439 ConfigureEncoder(video_encoder_config_.Copy());
6440
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01006441 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01006442 .WillByDefault(Return(SdpVideoFormat("AV1")));
6443 EXPECT_CALL(switch_callback,
6444 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
6445 Field(&SdpVideoFormat::name, "AV1"))));
6446
Henrik Boström381d1092020-05-12 18:49:07 +02006447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006448 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6449 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6450 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01006451 /*fraction_lost=*/0,
6452 /*rtt_ms=*/0,
6453 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006454 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01006455
6456 video_stream_encoder_->Stop();
6457}
6458
6459TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
6460 constexpr int kSufficientBitrateToNotDrop = 1000;
6461 constexpr int kDontCare = 100;
6462
6463 NiceMock<MockVideoEncoder> video_encoder;
6464 NiceMock<MockEncoderSelector> encoder_selector;
6465 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6466 video_send_config_.encoder_settings.encoder_switch_request_callback =
6467 &switch_callback;
6468 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6469 &video_encoder, &encoder_selector);
6470 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6471
6472 // Reset encoder for new configuration to take effect.
6473 ConfigureEncoder(video_encoder_config_.Copy());
6474
6475 // The VideoStreamEncoder needs some bitrate before it can start encoding,
6476 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
6477 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02006478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006479 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6480 /*stable_target_bitrate=*/
6481 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6482 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01006483 /*fraction_lost=*/0,
6484 /*rtt_ms=*/0,
6485 /*cwnd_reduce_ratio=*/0);
6486
6487 ON_CALL(video_encoder, Encode(_, _))
6488 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
6489 ON_CALL(encoder_selector, OnEncoderBroken())
6490 .WillByDefault(Return(SdpVideoFormat("AV2")));
6491
6492 rtc::Event encode_attempted;
6493 EXPECT_CALL(switch_callback,
6494 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
6495 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
6496 EXPECT_EQ(format.name, "AV2");
6497 encode_attempted.Set();
6498 });
6499
6500 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
6501 encode_attempted.Wait(3000);
6502
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006503 AdvanceTime(TimeDelta::Millis(0));
6504
philipel9b058032020-02-10 11:30:00 +01006505 video_stream_encoder_->Stop();
6506
6507 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
6508 // to it's factory, so in order for the encoder instance in the
6509 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
6510 // reset the |video_stream_encoder_| here.
6511 video_stream_encoder_.reset();
6512}
6513
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006514TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006515 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006516 const int kFrameWidth = 320;
6517 const int kFrameHeight = 180;
6518
6519 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006520 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006521 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006522 /*target_bitrate=*/rate,
6523 /*stable_target_bitrate=*/rate,
6524 /*link_allocation=*/rate,
6525 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006526 /*rtt_ms=*/0,
6527 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006528
6529 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006530 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006531 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6532 frame.set_rotation(kVideoRotation_270);
6533 video_source_.IncomingCapturedFrame(frame);
6534 WaitForEncodedFrame(timestamp_ms);
6535 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6536
6537 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006538 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02006539 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006540 /*target_bitrate=*/new_stable_rate,
6541 /*stable_target_bitrate=*/new_stable_rate,
6542 /*link_allocation=*/rate,
6543 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006544 /*rtt_ms=*/0,
6545 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006546 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6547 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
6548 video_stream_encoder_->Stop();
6549}
6550
6551TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006552 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006553 const int kFrameWidth = 320;
6554 const int kFrameHeight = 180;
6555
6556 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006557 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006559 /*target_bitrate=*/rate,
6560 /*stable_target_bitrate=*/rate,
6561 /*link_allocation=*/rate,
6562 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006563 /*rtt_ms=*/0,
6564 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006565
6566 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006567 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006568 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6569 frame.set_rotation(kVideoRotation_270);
6570 video_source_.IncomingCapturedFrame(frame);
6571 WaitForEncodedFrame(timestamp_ms);
6572 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6573
6574 // Set a higher target rate without changing the link_allocation. Should not
6575 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006576 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02006577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006578 /*target_bitrate=*/rate,
6579 /*stable_target_bitrate=*/new_stable_rate,
6580 /*link_allocation=*/rate,
6581 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006582 /*rtt_ms=*/0,
6583 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006584 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6585 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6586 video_stream_encoder_->Stop();
6587}
6588
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006589TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
6590 test::ScopedFieldTrials field_trials(
6591 "WebRTC-AutomaticAnimationDetectionScreenshare/"
6592 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
6593 const int kFramerateFps = 30;
6594 const int kWidth = 1920;
6595 const int kHeight = 1080;
6596 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
6597 // Works on screenshare mode.
6598 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
6599 // We rely on the automatic resolution adaptation, but we handle framerate
6600 // adaptation manually by mocking the stats proxy.
6601 video_source_.set_adaptation_enabled(true);
6602
6603 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02006604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006605 DataRate::BitsPerSec(kTargetBitrateBps),
6606 DataRate::BitsPerSec(kTargetBitrateBps),
6607 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006608 video_stream_encoder_->SetSource(&video_source_,
6609 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006610 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006611
6612 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6613 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
6614
6615 // Pass enough frames with the full update to trigger animation detection.
6616 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006617 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006618 frame.set_ntp_time_ms(timestamp_ms);
6619 frame.set_timestamp_us(timestamp_ms * 1000);
6620 video_source_.IncomingCapturedFrame(frame);
6621 WaitForEncodedFrame(timestamp_ms);
6622 }
6623
6624 // Resolution should be limited.
6625 rtc::VideoSinkWants expected;
6626 expected.max_framerate_fps = kFramerateFps;
6627 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006628 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006629
6630 // Pass one frame with no known update.
6631 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006632 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006633 frame.set_ntp_time_ms(timestamp_ms);
6634 frame.set_timestamp_us(timestamp_ms * 1000);
6635 frame.clear_update_rect();
6636
6637 video_source_.IncomingCapturedFrame(frame);
6638 WaitForEncodedFrame(timestamp_ms);
6639
6640 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006641 EXPECT_THAT(video_source_.sink_wants(),
6642 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006643
6644 video_stream_encoder_->Stop();
6645}
6646
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02006647TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
6648 const int kWidth = 720; // 540p adapted down.
6649 const int kHeight = 405;
6650 const int kNumFrames = 3;
6651 // Works on screenshare mode.
6652 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
6653 /*num_spatial_layers=*/2, /*screenshare=*/true);
6654
6655 video_source_.set_adaptation_enabled(true);
6656
6657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6658 DataRate::BitsPerSec(kTargetBitrateBps),
6659 DataRate::BitsPerSec(kTargetBitrateBps),
6660 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6661
6662 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6663
6664 // Pass enough frames with the full update to trigger animation detection.
6665 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006666 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02006667 frame.set_ntp_time_ms(timestamp_ms);
6668 frame.set_timestamp_us(timestamp_ms * 1000);
6669 video_source_.IncomingCapturedFrame(frame);
6670 WaitForEncodedFrame(timestamp_ms);
6671 }
6672
6673 video_stream_encoder_->Stop();
6674}
6675
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07006676TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
6677 const float downscale_factors[] = {4.0, 2.0, 1.0};
6678 const int number_layers =
6679 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
6680 VideoEncoderConfig config;
6681 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
6682 for (int i = 0; i < number_layers; ++i) {
6683 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
6684 config.simulcast_layers[i].active = true;
6685 }
6686 config.video_stream_factory =
6687 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
6688 "VP8", /*max qp*/ 56, /*screencast*/ false,
6689 /*screenshare enabled*/ false);
6690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6691 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6692 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6693 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
6694
6695 // First initialization.
6696 // Encoder should be initialized. Next frame should be key frame.
6697 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
6698 sink_.SetNumExpectedLayers(number_layers);
6699 int64_t timestamp_ms = kFrameIntervalMs;
6700 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
6701 WaitForEncodedFrame(timestamp_ms);
6702 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
6703 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
6704 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6705 VideoFrameType::kVideoFrameKey,
6706 VideoFrameType::kVideoFrameKey}));
6707
6708 // Disable top layer.
6709 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
6710 config.simulcast_layers[number_layers - 1].active = false;
6711 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
6712 sink_.SetNumExpectedLayers(number_layers - 1);
6713 timestamp_ms += kFrameIntervalMs;
6714 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
6715 WaitForEncodedFrame(timestamp_ms);
6716 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
6717 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
6718 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6719 VideoFrameType::kVideoFrameDelta,
6720 VideoFrameType::kVideoFrameDelta}));
6721
6722 // Re-enable top layer.
6723 // Encoder should be re-initialized. Next frame should be key frame.
6724 config.simulcast_layers[number_layers - 1].active = true;
6725 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
6726 sink_.SetNumExpectedLayers(number_layers);
6727 timestamp_ms += kFrameIntervalMs;
6728 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
6729 WaitForEncodedFrame(timestamp_ms);
6730 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
6731 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
6732 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6733 VideoFrameType::kVideoFrameKey,
6734 VideoFrameType::kVideoFrameKey}));
6735
6736 // Top layer max rate change.
6737 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
6738 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
6739 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
6740 sink_.SetNumExpectedLayers(number_layers);
6741 timestamp_ms += kFrameIntervalMs;
6742 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
6743 WaitForEncodedFrame(timestamp_ms);
6744 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
6745 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
6746 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6747 VideoFrameType::kVideoFrameDelta,
6748 VideoFrameType::kVideoFrameDelta}));
6749
6750 // Top layer resolution change.
6751 // Encoder should be re-initialized. Next frame should be key frame.
6752 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
6753 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
6754 sink_.SetNumExpectedLayers(number_layers);
6755 timestamp_ms += kFrameIntervalMs;
6756 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
6757 WaitForEncodedFrame(timestamp_ms);
6758 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
6759 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
6760 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6761 VideoFrameType::kVideoFrameKey,
6762 VideoFrameType::kVideoFrameKey}));
6763 video_stream_encoder_->Stop();
6764}
6765
perkj26091b12016-09-01 01:17:40 -07006766} // namespace webrtc