blob: ced05ec91644c43db59a173ac6cd7efd99ed30ad [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020040#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020043#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010044#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020045#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020060using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020062using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020063using ::testing::Ge;
64using ::testing::Gt;
65using ::testing::Le;
66using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010067using ::testing::Matcher;
68using ::testing::NiceMock;
69using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010070using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020071using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080072
perkj803d97f2016-11-01 11:45:46 -070073namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010075const int kQpLow = 1;
76const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020077const int kMinFramerateFps = 2;
78const int kMinBalancedFramerateFps = 7;
79const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080080const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020082const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010083const uint32_t kSimulcastTargetBitrateBps = 3150000;
84const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080085const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070086const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020088const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020089const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
91const VideoEncoder::ResolutionBitrateLimits
92 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020094uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
95 0x00, 0x00, 0x03, 0x03, 0xF4,
96 0x05, 0x03, 0xC7, 0xE0, 0x1B,
97 0x41, 0x10, 0x8D, 0x00};
98
perkj803d97f2016-11-01 11:45:46 -070099class TestBuffer : public webrtc::I420Buffer {
100 public:
101 TestBuffer(rtc::Event* event, int width, int height)
102 : I420Buffer(width, height), event_(event) {}
103
104 private:
105 friend class rtc::RefCountedObject<TestBuffer>;
106 ~TestBuffer() override {
107 if (event_)
108 event_->Set();
109 }
110 rtc::Event* const event_;
111};
112
Noah Richards51db4212019-06-12 06:59:12 -0700113// A fake native buffer that can't be converted to I420.
114class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
115 public:
116 FakeNativeBuffer(rtc::Event* event, int width, int height)
117 : event_(event), width_(width), height_(height) {}
118 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
119 int width() const override { return width_; }
120 int height() const override { return height_; }
121 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
122 return nullptr;
123 }
124
125 private:
126 friend class rtc::RefCountedObject<FakeNativeBuffer>;
127 ~FakeNativeBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132 const int width_;
133 const int height_;
134};
135
Evan Shrubsole895556e2020-10-05 09:15:13 +0200136// A fake native buffer that is backed by an NV12 buffer.
137class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
140 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
141
142 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
143 int width() const override { return nv12_buffer_->width(); }
144 int height() const override { return nv12_buffer_->height(); }
145 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
146 return nv12_buffer_->ToI420();
147 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200148 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
149 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
150 if (absl::c_find(types, Type::kNV12) != types.end()) {
151 return nv12_buffer_;
152 }
153 return nullptr;
154 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200155 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
156
157 private:
158 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
159 ~FakeNV12NativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
164 rtc::Event* const event_;
165};
166
Niels Möller7dc26b72017-12-06 10:27:48 +0100167class CpuOveruseDetectorProxy : public OveruseFrameDetector {
168 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200169 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
170 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200171 last_target_framerate_fps_(-1),
172 framerate_updated_event_(true /* manual_reset */,
173 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100174 virtual ~CpuOveruseDetectorProxy() {}
175
176 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200177 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100178 last_target_framerate_fps_ = framerate_fps;
179 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200180 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100181 }
182
183 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200184 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100185 return last_target_framerate_fps_;
186 }
187
Niels Möller4db138e2018-04-19 09:04:13 +0200188 CpuOveruseOptions GetOptions() { return options_; }
189
Henrik Boström381d1092020-05-12 18:49:07 +0200190 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
191
Niels Möller7dc26b72017-12-06 10:27:48 +0100192 private:
Markus Handella3765182020-07-08 13:13:32 +0200193 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100194 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200195 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100196};
197
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200198class FakeVideoSourceRestrictionsListener
199 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200200 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200201 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200202 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200203 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200204 RTC_DCHECK(was_restrictions_updated_);
205 }
206
207 rtc::Event* restrictions_updated_event() {
208 return &restrictions_updated_event_;
209 }
210
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200211 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200212 void OnVideoSourceRestrictionsUpdated(
213 VideoSourceRestrictions restrictions,
214 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200215 rtc::scoped_refptr<Resource> reason,
216 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200217 was_restrictions_updated_ = true;
218 restrictions_updated_event_.Set();
219 }
220
221 private:
222 bool was_restrictions_updated_;
223 rtc::Event restrictions_updated_event_;
224};
225
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200226auto WantsFps(Matcher<int> fps_matcher) {
227 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
228 fps_matcher);
229}
230
231auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
232 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
233 AllOf(max_pixel_matcher, Gt(0)));
234}
235
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200236auto ResolutionMax() {
237 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
240 Eq(absl::nullopt)));
241}
242
243auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245}
246
247auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200249}
250
251auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200253}
254
255auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200256 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200257}
258
259auto FpsMaxResolutionMax() {
260 return AllOf(FpsMax(), ResolutionMax());
261}
262
263auto UnlimitedSinkWants() {
264 return AllOf(FpsUnlimited(), ResolutionMax());
265}
266
267auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
268 Matcher<int> fps_range_matcher;
269
270 if (last_frame_pixels <= 320 * 240) {
271 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200272 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200273 fps_range_matcher = AllOf(Ge(10), Le(15));
274 } else if (last_frame_pixels <= 640 * 480) {
275 fps_range_matcher = Ge(15);
276 } else {
277 fps_range_matcher = Eq(kDefaultFramerate);
278 }
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_range_matcher);
281}
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
284 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
285 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
286}
287
288auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
289 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
290}
291
292auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
293 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
294}
295
296auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
299}
300
301auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
304}
305
306auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
307 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
308 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
309}
310
311auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
312 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
313 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
314}
315
mflodmancc3d4422017-08-03 08:27:51 -0700316class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700317 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200318 VideoStreamEncoderUnderTest(TimeController* time_controller,
319 TaskQueueFactory* task_queue_factory,
320 SendStatisticsProxy* stats_proxy,
321 const VideoStreamEncoderSettings& settings)
322 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100323 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200324 stats_proxy,
325 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200326 std::unique_ptr<OveruseFrameDetector>(
327 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100328 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100329 task_queue_factory),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200330 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200331 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200332 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200333 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200334 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200335 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200336 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200337 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100338 }
perkj803d97f2016-11-01 11:45:46 -0700339
Henrik Boström381d1092020-05-12 18:49:07 +0200340 void SetSourceAndWaitForRestrictionsUpdated(
341 rtc::VideoSourceInterface<VideoFrame>* source,
342 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200343 FakeVideoSourceRestrictionsListener listener;
344 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200345 SetSource(source, degradation_preference);
346 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200347 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200348 }
349
350 void SetSourceAndWaitForFramerateUpdated(
351 rtc::VideoSourceInterface<VideoFrame>* source,
352 const DegradationPreference& degradation_preference) {
353 overuse_detector_proxy_->framerate_updated_event()->Reset();
354 SetSource(source, degradation_preference);
355 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
356 }
357
358 void OnBitrateUpdatedAndWaitForManagedResources(
359 DataRate target_bitrate,
360 DataRate stable_target_bitrate,
361 DataRate link_allocation,
362 uint8_t fraction_lost,
363 int64_t round_trip_time_ms,
364 double cwnd_reduce_ratio) {
365 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
366 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
367 // Bitrate is updated on the encoder queue.
368 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200369 }
370
kthelgason2fc52542017-03-03 00:24:41 -0800371 // This is used as a synchronisation mechanism, to make sure that the
372 // encoder queue is not blocked before we start sending it frames.
373 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100374 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200375 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800376 ASSERT_TRUE(event.Wait(5000));
377 }
378
Henrik Boström91aa7322020-04-28 12:24:33 +0200379 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200380 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200381 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200382 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200383 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200384 event.Set();
385 });
386 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200387 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200388 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200389
Henrik Boström91aa7322020-04-28 12:24:33 +0200390 void TriggerCpuUnderuse() {
391 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200392 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200393 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200394 event.Set();
395 });
396 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200397 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200398 }
kthelgason876222f2016-11-29 01:44:11 -0800399
Henrik Boström91aa7322020-04-28 12:24:33 +0200400 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200401 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200402 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200403 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200404 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200405 event.Set();
406 });
407 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200408 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200409 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 }
419
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100421 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200422 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
423 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200424 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700425};
426
Noah Richards51db4212019-06-12 06:59:12 -0700427// Simulates simulcast behavior and makes highest stream resolutions divisible
428// by 4.
429class CroppingVideoStreamFactory
430 : public VideoEncoderConfig::VideoStreamFactoryInterface {
431 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200432 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700433
434 private:
435 std::vector<VideoStream> CreateEncoderStreams(
436 int width,
437 int height,
438 const VideoEncoderConfig& encoder_config) override {
439 std::vector<VideoStream> streams = test::CreateVideoStreams(
440 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700441 return streams;
442 }
Noah Richards51db4212019-06-12 06:59:12 -0700443};
444
sprangb1ca0732017-02-01 08:38:12 -0800445class AdaptingFrameForwarder : public test::FrameForwarder {
446 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447 explicit AdaptingFrameForwarder(TimeController* time_controller)
448 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700449 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800450
451 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200452 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800453 adaptation_enabled_ = enabled;
454 }
455
asaperssonfab67072017-04-04 05:51:49 -0700456 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200457 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800458 return adaptation_enabled_;
459 }
460
asapersson09f05612017-05-15 23:40:18 -0700461 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200462 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700463 return last_wants_;
464 }
465
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200466 absl::optional<int> last_sent_width() const { return last_width_; }
467 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800468
sprangb1ca0732017-02-01 08:38:12 -0800469 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200470 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
471 time_controller_->AdvanceTime(TimeDelta::Millis(0));
472
sprangb1ca0732017-02-01 08:38:12 -0800473 int cropped_width = 0;
474 int cropped_height = 0;
475 int out_width = 0;
476 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700477 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200478 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
479 << "w=" << video_frame.width()
480 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700481 if (adapter_.AdaptFrameResolution(
482 video_frame.width(), video_frame.height(),
483 video_frame.timestamp_us() * 1000, &cropped_width,
484 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100485 VideoFrame adapted_frame =
486 VideoFrame::Builder()
487 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
488 nullptr, out_width, out_height))
489 .set_timestamp_rtp(99)
490 .set_timestamp_ms(99)
491 .set_rotation(kVideoRotation_0)
492 .build();
sprangc5d62e22017-04-02 23:53:04 -0700493 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100494 if (video_frame.has_update_rect()) {
495 adapted_frame.set_update_rect(
496 video_frame.update_rect().ScaleWithFrame(
497 video_frame.width(), video_frame.height(), 0, 0,
498 video_frame.width(), video_frame.height(), out_width,
499 out_height));
500 }
sprangc5d62e22017-04-02 23:53:04 -0700501 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800502 last_width_.emplace(adapted_frame.width());
503 last_height_.emplace(adapted_frame.height());
504 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200505 last_width_ = absl::nullopt;
506 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700507 }
sprangb1ca0732017-02-01 08:38:12 -0800508 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200509 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800510 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800511 last_width_.emplace(video_frame.width());
512 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800513 }
514 }
515
516 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
517 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200518 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200519 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100520 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200521 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800522 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200523
524 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800525 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200526 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
527 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200528 absl::optional<int> last_width_;
529 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800530};
sprangc5d62e22017-04-02 23:53:04 -0700531
Niels Möller213618e2018-07-24 09:29:58 +0200532// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700533class MockableSendStatisticsProxy : public SendStatisticsProxy {
534 public:
535 MockableSendStatisticsProxy(Clock* clock,
536 const VideoSendStream::Config& config,
537 VideoEncoderConfig::ContentType content_type)
538 : SendStatisticsProxy(clock, config, content_type) {}
539
540 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200541 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700542 if (mock_stats_)
543 return *mock_stats_;
544 return SendStatisticsProxy::GetStats();
545 }
546
Niels Möller213618e2018-07-24 09:29:58 +0200547 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200548 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200549 if (mock_stats_)
550 return mock_stats_->input_frame_rate;
551 return SendStatisticsProxy::GetInputFrameRate();
552 }
sprangc5d62e22017-04-02 23:53:04 -0700553 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200554 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700555 mock_stats_.emplace(stats);
556 }
557
558 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200559 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700560 mock_stats_.reset();
561 }
562
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200563 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
564 on_frame_dropped_ = std::move(callback);
565 }
566
sprangc5d62e22017-04-02 23:53:04 -0700567 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200568 void OnFrameDropped(DropReason reason) override {
569 SendStatisticsProxy::OnFrameDropped(reason);
570 if (on_frame_dropped_)
571 on_frame_dropped_(reason);
572 }
573
Markus Handella3765182020-07-08 13:13:32 +0200574 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200575 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200576 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700577};
578
philipel9b058032020-02-10 11:30:00 +0100579class MockEncoderSelector
580 : public VideoEncoderFactory::EncoderSelectorInterface {
581 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200582 MOCK_METHOD(void,
583 OnCurrentEncoder,
584 (const SdpVideoFormat& format),
585 (override));
586 MOCK_METHOD(absl::optional<SdpVideoFormat>,
587 OnAvailableBitrate,
588 (const DataRate& rate),
589 (override));
590 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100591};
592
perkj803d97f2016-11-01 11:45:46 -0700593} // namespace
594
mflodmancc3d4422017-08-03 08:27:51 -0700595class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700596 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200597 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700598
mflodmancc3d4422017-08-03 08:27:51 -0700599 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700600 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700601 codec_width_(320),
602 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200603 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200604 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200605 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700606 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700608 video_send_config_,
609 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700611
612 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700613 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700614 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200615 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800616 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200617 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200618 video_send_config_.rtp.payload_name = "FAKE";
619 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700620
Per512ecb32016-09-23 15:52:06 +0200621 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200622 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200623 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
624 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
625 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100626 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700627
Niels Möllerf1338562018-04-26 09:51:47 +0200628 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800629 }
630
Niels Möllerf1338562018-04-26 09:51:47 +0200631 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700632 if (video_stream_encoder_)
633 video_stream_encoder_->Stop();
634 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200635 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
636 video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700637 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
638 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700639 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700640 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
641 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200642 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700643 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800644 }
645
Per Kjellanderdcef6412020-10-07 15:09:05 +0200646 void ResetEncoder(
647 const std::string& payload_name,
648 size_t num_streams,
649 size_t num_temporal_layers,
650 unsigned char num_spatial_layers,
651 bool screenshare,
652 VideoStreamEncoderSettings::BitrateAllocationCallbackType
653 allocation_cb_type =
654 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
655 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200656 video_send_config_.rtp.payload_name = payload_name;
Per Kjellanderdcef6412020-10-07 15:09:05 +0200657 video_send_config_.encoder_settings.allocation_cb_type = allocation_cb_type;
asapersson5f7226f2016-11-25 04:37:00 -0800658
659 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200660 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
661 num_streams, &video_encoder_config);
662 for (auto& layer : video_encoder_config.simulcast_layers) {
663 layer.num_temporal_layers = num_temporal_layers;
664 layer.max_framerate = kDefaultFramerate;
665 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100666 video_encoder_config.max_bitrate_bps =
667 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700668 video_encoder_config.content_type =
669 screenshare ? VideoEncoderConfig::ContentType::kScreen
670 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700671 if (payload_name == "VP9") {
672 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
673 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200674 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700675 video_encoder_config.encoder_specific_settings =
676 new rtc::RefCountedObject<
677 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
678 }
Niels Möllerf1338562018-04-26 09:51:47 +0200679 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700680 }
681
sprang57c2fff2017-01-16 06:24:02 -0800682 VideoFrame CreateFrame(int64_t ntp_time_ms,
683 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100684 VideoFrame frame =
685 VideoFrame::Builder()
686 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
687 destruction_event, codec_width_, codec_height_))
688 .set_timestamp_rtp(99)
689 .set_timestamp_ms(99)
690 .set_rotation(kVideoRotation_0)
691 .build();
sprang57c2fff2017-01-16 06:24:02 -0800692 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700693 return frame;
694 }
695
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100696 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
697 rtc::Event* destruction_event,
698 int offset_x) const {
699 VideoFrame frame =
700 VideoFrame::Builder()
701 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
702 destruction_event, codec_width_, codec_height_))
703 .set_timestamp_rtp(99)
704 .set_timestamp_ms(99)
705 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100706 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100707 .build();
708 frame.set_ntp_time_ms(ntp_time_ms);
709 return frame;
710 }
711
sprang57c2fff2017-01-16 06:24:02 -0800712 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100713 VideoFrame frame =
714 VideoFrame::Builder()
715 .set_video_frame_buffer(
716 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
717 .set_timestamp_rtp(99)
718 .set_timestamp_ms(99)
719 .set_rotation(kVideoRotation_0)
720 .build();
sprang57c2fff2017-01-16 06:24:02 -0800721 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700722 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700723 return frame;
724 }
725
Evan Shrubsole895556e2020-10-05 09:15:13 +0200726 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
727 VideoFrame frame =
728 VideoFrame::Builder()
729 .set_video_frame_buffer(NV12Buffer::Create(width, height))
730 .set_timestamp_rtp(99)
731 .set_timestamp_ms(99)
732 .set_rotation(kVideoRotation_0)
733 .build();
734 frame.set_ntp_time_ms(ntp_time_ms);
735 frame.set_timestamp_us(ntp_time_ms * 1000);
736 return frame;
737 }
738
Noah Richards51db4212019-06-12 06:59:12 -0700739 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
740 rtc::Event* destruction_event,
741 int width,
742 int height) const {
743 VideoFrame frame =
744 VideoFrame::Builder()
745 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
746 destruction_event, width, height))
747 .set_timestamp_rtp(99)
748 .set_timestamp_ms(99)
749 .set_rotation(kVideoRotation_0)
750 .build();
751 frame.set_ntp_time_ms(ntp_time_ms);
752 return frame;
753 }
754
Evan Shrubsole895556e2020-10-05 09:15:13 +0200755 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
756 rtc::Event* destruction_event,
757 int width,
758 int height) const {
759 VideoFrame frame = VideoFrame::Builder()
760 .set_video_frame_buffer(
761 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
762 destruction_event, width, height))
763 .set_timestamp_rtp(99)
764 .set_timestamp_ms(99)
765 .set_rotation(kVideoRotation_0)
766 .build();
767 frame.set_ntp_time_ms(ntp_time_ms);
768 return frame;
769 }
770
Noah Richards51db4212019-06-12 06:59:12 -0700771 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
772 rtc::Event* destruction_event) const {
773 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
774 codec_height_);
775 }
776
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100777 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100779 DataRate::BitsPerSec(kTargetBitrateBps),
780 DataRate::BitsPerSec(kTargetBitrateBps),
781 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100782
783 video_source_.IncomingCapturedFrame(
784 CreateFrame(1, codec_width_, codec_height_));
785 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200786 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100787 }
788
sprang4847ae62017-06-27 07:06:52 -0700789 void WaitForEncodedFrame(int64_t expected_ntp_time) {
790 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200791 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700792 }
793
794 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
795 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200796 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700797 return ok;
798 }
799
800 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
801 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200802 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700803 }
804
805 void ExpectDroppedFrame() {
806 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200807 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700808 }
809
810 bool WaitForFrame(int64_t timeout_ms) {
811 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200812 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700813 return ok;
814 }
815
perkj26091b12016-09-01 01:17:40 -0700816 class TestEncoder : public test::FakeEncoder {
817 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200818 explicit TestEncoder(TimeController* time_controller)
819 : FakeEncoder(time_controller->GetClock()),
820 time_controller_(time_controller) {
821 RTC_DCHECK(time_controller_);
822 }
perkj26091b12016-09-01 01:17:40 -0700823
asaperssonfab67072017-04-04 05:51:49 -0700824 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200825 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700826 return config_;
827 }
828
829 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200830 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700831 block_next_encode_ = true;
832 }
833
Erik Språngaed30702018-11-05 12:57:17 +0100834 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200835 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200836 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100837 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100838 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100839 info.scaling_settings = VideoEncoder::ScalingSettings(
840 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100841 }
842 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100843 for (int i = 0; i < kMaxSpatialLayers; ++i) {
844 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100845 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100846 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100847 for (int tid = 0; tid < num_layers; ++tid)
848 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 }
850 }
Erik Språngaed30702018-11-05 12:57:17 +0100851 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200852
853 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100854 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200855 info.apply_alignment_to_all_simulcast_layers =
856 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200857 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100858 return info;
kthelgason876222f2016-11-29 01:44:11 -0800859 }
860
Erik Språngb7cb7b52019-02-26 15:52:33 +0100861 int32_t RegisterEncodeCompleteCallback(
862 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200863 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100864 encoded_image_callback_ = callback;
865 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
866 }
867
perkjfa10b552016-10-02 23:45:26 -0700868 void ContinueEncode() { continue_encode_event_.Set(); }
869
870 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
871 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200872 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700873 EXPECT_EQ(timestamp_, timestamp);
874 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
875 }
876
kthelgason2fc52542017-03-03 00:24:41 -0800877 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200878 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800879 quality_scaling_ = b;
880 }
kthelgasonad9010c2017-02-14 00:46:51 -0800881
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100882 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200883 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100884 requested_resolution_alignment_ = requested_resolution_alignment;
885 }
886
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200887 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
888 MutexLock lock(&local_mutex_);
889 apply_alignment_to_all_simulcast_layers_ = b;
890 }
891
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100892 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200893 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100894 is_hardware_accelerated_ = is_hardware_accelerated;
895 }
896
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100897 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
898 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200899 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100900 temporal_layers_supported_[spatial_idx] = supported;
901 }
902
Sergey Silkin6456e352019-07-08 17:56:40 +0200903 void SetResolutionBitrateLimits(
904 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200906 resolution_bitrate_limits_ = thresholds;
907 }
908
sprangfe627f32017-03-29 08:24:59 -0700909 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200910 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700911 force_init_encode_failed_ = force_failure;
912 }
913
Niels Möller6bb5ab92019-01-11 11:11:10 +0100914 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200915 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100916 rate_factor_ = rate_factor;
917 }
918
Erik Språngd7329ca2019-02-21 21:19:53 +0100919 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200920 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100921 return last_framerate_;
922 }
923
Erik Språngd7329ca2019-02-21 21:19:53 +0100924 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200925 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100926 return last_update_rect_;
927 }
928
Niels Möller87e2d782019-03-07 10:18:23 +0100929 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200930 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100931 return last_frame_types_;
932 }
933
934 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100935 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100936 keyframe ? VideoFrameType::kVideoFrameKey
937 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100938 {
Markus Handella3765182020-07-08 13:13:32 +0200939 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100940 last_frame_types_ = frame_type;
941 }
Niels Möllerb859b322019-03-07 12:40:01 +0100942 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100943 }
944
Erik Språngb7cb7b52019-02-26 15:52:33 +0100945 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200946 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200947 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100948 }
949
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200950 void SetEncodedImageData(
951 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200952 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200953 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200954 }
955
Erik Språngd7329ca2019-02-21 21:19:53 +0100956 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200957 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100958 expect_null_frame_ = true;
959 }
960
Erik Språng5056af02019-09-02 15:53:11 +0200961 absl::optional<VideoEncoder::RateControlParameters>
962 GetAndResetLastRateControlSettings() {
963 auto settings = last_rate_control_settings_;
964 last_rate_control_settings_.reset();
965 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100966 }
967
Evan Shrubsole895556e2020-10-05 09:15:13 +0200968 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
969 MutexLock lock(&local_mutex_);
970 return last_input_pixel_format_;
971 }
972
Sergey Silkin5ee69672019-07-02 14:18:34 +0200973 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200974 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200975 return num_encoder_initializations_;
976 }
977
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200978 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200979 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200980 return num_set_rates_;
981 }
982
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200983 VideoCodec video_codec() const {
984 MutexLock lock(&local_mutex_);
985 return video_codec_;
986 }
987
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200988 void SetPreferredPixelFormats(
989 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
990 pixel_formats) {
991 MutexLock lock(&local_mutex_);
992 preferred_pixel_formats_ = std::move(pixel_formats);
993 }
994
perkjfa10b552016-10-02 23:45:26 -0700995 private:
perkj26091b12016-09-01 01:17:40 -0700996 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100997 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700998 bool block_encode;
999 {
Markus Handella3765182020-07-08 13:13:32 +02001000 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001001 if (expect_null_frame_) {
1002 EXPECT_EQ(input_image.timestamp(), 0u);
1003 EXPECT_EQ(input_image.width(), 1);
1004 last_frame_types_ = *frame_types;
1005 expect_null_frame_ = false;
1006 } else {
1007 EXPECT_GT(input_image.timestamp(), timestamp_);
1008 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1009 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1010 }
perkj26091b12016-09-01 01:17:40 -07001011
1012 timestamp_ = input_image.timestamp();
1013 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001014 last_input_width_ = input_image.width();
1015 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001016 block_encode = block_next_encode_;
1017 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001018 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001019 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001020 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001021 }
Niels Möllerb859b322019-03-07 12:40:01 +01001022 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001023 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001024 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001025
perkj26091b12016-09-01 01:17:40 -07001026 return result;
1027 }
1028
Niels Möller08ae7ce2020-09-23 15:58:12 +02001029 CodecSpecificInfo EncodeHook(
1030 EncodedImage& encoded_image,
1031 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001032 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001033 {
1034 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001035 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001036 }
1037 MutexLock lock(&local_mutex_);
1038 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001039 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001040 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001041 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001042 }
1043
sprangfe627f32017-03-29 08:24:59 -07001044 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001045 const Settings& settings) override {
1046 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001047
Markus Handella3765182020-07-08 13:13:32 +02001048 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001049 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001050
1051 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001052 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001053
Erik Språng82fad3d2018-03-21 09:57:23 +01001054 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001055 // Simulate setting up temporal layers, in order to validate the life
1056 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001057 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001058 frame_buffer_controller_ =
1059 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001060 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001061 if (force_init_encode_failed_) {
1062 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001063 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001064 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001065
Erik Språngb7cb7b52019-02-26 15:52:33 +01001066 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001067 return res;
1068 }
1069
Erik Språngb7cb7b52019-02-26 15:52:33 +01001070 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001071 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001072 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1073 initialized_ = EncoderState::kUninitialized;
1074 return FakeEncoder::Release();
1075 }
1076
Erik Språng16cb8f52019-04-12 13:59:09 +02001077 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001078 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001079 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001080 VideoBitrateAllocation adjusted_rate_allocation;
1081 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1082 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001083 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001084 adjusted_rate_allocation.SetBitrate(
1085 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001086 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001087 rate_factor_));
1088 }
1089 }
1090 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001091 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001092 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001093 RateControlParameters adjusted_paramters = parameters;
1094 adjusted_paramters.bitrate = adjusted_rate_allocation;
1095 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001096 }
1097
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001098 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001099 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001100 enum class EncoderState {
1101 kUninitialized,
1102 kInitializationFailed,
1103 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001104 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1105 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001106 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001107 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1108 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1109 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1110 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1111 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1112 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001113 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1114 false;
Markus Handella3765182020-07-08 13:13:32 +02001115 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001116 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1117 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001118 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001119 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001120 absl::optional<bool>
1121 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001122 local_mutex_);
1123 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1124 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1125 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001126 absl::optional<VideoEncoder::RateControlParameters>
1127 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001128 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1129 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001130 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001131 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001132 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1133 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001134 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001135 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001136 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001137 RTC_GUARDED_BY(local_mutex_);
1138 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001139 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001140 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1141 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001142 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1143 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001144 };
1145
mflodmancc3d4422017-08-03 08:27:51 -07001146 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001147 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001148 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1149 : time_controller_(time_controller), test_encoder_(test_encoder) {
1150 RTC_DCHECK(time_controller_);
1151 }
perkj26091b12016-09-01 01:17:40 -07001152
perkj26091b12016-09-01 01:17:40 -07001153 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001154 EXPECT_TRUE(
1155 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1156 }
1157
1158 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1159 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001160 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001161 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001162 return false;
perkj26091b12016-09-01 01:17:40 -07001163 {
Markus Handella3765182020-07-08 13:13:32 +02001164 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001165 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001166 }
1167 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001168 return true;
perkj26091b12016-09-01 01:17:40 -07001169 }
1170
sprangb1ca0732017-02-01 08:38:12 -08001171 void WaitForEncodedFrame(uint32_t expected_width,
1172 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001173 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001174 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001175 }
1176
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001177 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001178 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001179 uint32_t width = 0;
1180 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001181 {
Markus Handella3765182020-07-08 13:13:32 +02001182 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001183 width = last_width_;
1184 height = last_height_;
1185 }
1186 EXPECT_EQ(expected_height, height);
1187 EXPECT_EQ(expected_width, width);
1188 }
1189
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001190 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1191 VideoRotation rotation;
1192 {
Markus Handella3765182020-07-08 13:13:32 +02001193 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001194 rotation = last_rotation_;
1195 }
1196 EXPECT_EQ(expected_rotation, rotation);
1197 }
1198
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001199 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001200
sprangc5d62e22017-04-02 23:53:04 -07001201 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001202 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1203 bool ret = encoded_frame_event_.Wait(timeout_ms);
1204 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1205 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001206 }
1207
perkj26091b12016-09-01 01:17:40 -07001208 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001209 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001210 expect_frames_ = false;
1211 }
1212
asaperssonfab67072017-04-04 05:51:49 -07001213 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001214 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001215 return number_of_reconfigurations_;
1216 }
1217
asaperssonfab67072017-04-04 05:51:49 -07001218 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001219 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001220 return min_transmit_bitrate_bps_;
1221 }
1222
Erik Språngd7329ca2019-02-21 21:19:53 +01001223 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001224 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001225 num_expected_layers_ = num_layers;
1226 }
1227
Erik Språngb7cb7b52019-02-26 15:52:33 +01001228 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001229 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001230 return last_capture_time_ms_;
1231 }
1232
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001233 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001234 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001235 return std::move(last_encoded_image_data_);
1236 }
1237
Per Kjellanderdcef6412020-10-07 15:09:05 +02001238 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1239 MutexLock lock(&mutex_);
1240 return last_bitrate_allocation_;
1241 }
1242
1243 int number_of_bitrate_allocations() const {
1244 MutexLock lock(&mutex_);
1245 return number_of_bitrate_allocations_;
1246 }
1247
Per Kjellandera9434842020-10-15 17:53:22 +02001248 VideoLayersAllocation GetLastVideoLayersAllocation() {
1249 MutexLock lock(&mutex_);
1250 return last_layers_allocation_;
1251 }
1252
1253 int number_of_layers_allocations() const {
1254 MutexLock lock(&mutex_);
1255 return number_of_layers_allocations_;
1256 }
1257
perkj26091b12016-09-01 01:17:40 -07001258 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001259 Result OnEncodedImage(
1260 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001261 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001262 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001263 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001264 last_encoded_image_data_ = std::vector<uint8_t>(
1265 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001266 uint32_t timestamp = encoded_image.Timestamp();
1267 if (last_timestamp_ != timestamp) {
1268 num_received_layers_ = 1;
1269 } else {
1270 ++num_received_layers_;
1271 }
1272 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001273 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001274 last_width_ = encoded_image._encodedWidth;
1275 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001276 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001277 if (num_received_layers_ == num_expected_layers_) {
1278 encoded_frame_event_.Set();
1279 }
sprangb1ca0732017-02-01 08:38:12 -08001280 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001281 }
1282
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001283 void OnEncoderConfigurationChanged(
1284 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001285 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001286 VideoEncoderConfig::ContentType content_type,
1287 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001288 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001289 ++number_of_reconfigurations_;
1290 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1291 }
1292
Per Kjellanderdcef6412020-10-07 15:09:05 +02001293 void OnBitrateAllocationUpdated(
1294 const VideoBitrateAllocation& allocation) override {
1295 MutexLock lock(&mutex_);
1296 ++number_of_bitrate_allocations_;
1297 last_bitrate_allocation_ = allocation;
1298 }
1299
Per Kjellandera9434842020-10-15 17:53:22 +02001300 void OnVideoLayersAllocationUpdated(
1301 VideoLayersAllocation allocation) override {
1302 MutexLock lock(&mutex_);
1303 ++number_of_layers_allocations_;
1304 last_layers_allocation_ = allocation;
1305 rtc::StringBuilder log;
1306 for (const auto& layer : allocation.active_spatial_layers) {
1307 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1308 << "[";
1309 for (const auto target_bitrate :
1310 layer.target_bitrate_per_temporal_layer) {
1311 log << target_bitrate.kbps() << ",";
1312 }
1313 log << "]";
1314 }
1315 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1316 }
1317
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001318 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001319 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001320 TestEncoder* test_encoder_;
1321 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001322 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001323 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001324 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001325 uint32_t last_height_ = 0;
1326 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001327 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001328 size_t num_expected_layers_ = 1;
1329 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001330 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001331 int number_of_reconfigurations_ = 0;
1332 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001333 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1334 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001335 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1336 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001337 };
1338
Sergey Silkin5ee69672019-07-02 14:18:34 +02001339 class VideoBitrateAllocatorProxyFactory
1340 : public VideoBitrateAllocatorFactory {
1341 public:
1342 VideoBitrateAllocatorProxyFactory()
1343 : bitrate_allocator_factory_(
1344 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1345
1346 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1347 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001348 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001349 codec_config_ = codec;
1350 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1351 }
1352
1353 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001354 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001355 return codec_config_;
1356 }
1357
1358 private:
1359 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1360
Markus Handella3765182020-07-08 13:13:32 +02001361 mutable Mutex mutex_;
1362 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001363 };
1364
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001365 Clock* clock() { return time_controller_.GetClock(); }
1366 void AdvanceTime(TimeDelta duration) {
1367 time_controller_.AdvanceTime(duration);
1368 }
1369
1370 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1371
1372 protected:
1373 virtual TaskQueueFactory* GetTaskQueueFactory() {
1374 return time_controller_.GetTaskQueueFactory();
1375 }
1376
1377 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001378 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001379 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001380 int codec_width_;
1381 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001382 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001383 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001384 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001385 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001386 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001387 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001388 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001389 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001390};
1391
mflodmancc3d4422017-08-03 08:27:51 -07001392TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001393 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001394 DataRate::BitsPerSec(kTargetBitrateBps),
1395 DataRate::BitsPerSec(kTargetBitrateBps),
1396 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001397 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001398 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001399 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001400 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001401 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001402}
1403
mflodmancc3d4422017-08-03 08:27:51 -07001404TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001405 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001406 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001407 // The encoder will cache up to one frame for a short duration. Adding two
1408 // frames means that the first frame will be dropped and the second frame will
1409 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001410 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001411 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001412 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001413 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001414
Henrik Boström381d1092020-05-12 18:49:07 +02001415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001416 DataRate::BitsPerSec(kTargetBitrateBps),
1417 DataRate::BitsPerSec(kTargetBitrateBps),
1418 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001419
Sebastian Janssona3177052018-04-10 13:05:49 +02001420 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001421 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001422 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1423
1424 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001425 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001426}
1427
mflodmancc3d4422017-08-03 08:27:51 -07001428TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001429 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001430 DataRate::BitsPerSec(kTargetBitrateBps),
1431 DataRate::BitsPerSec(kTargetBitrateBps),
1432 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001433 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001434 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001435
Henrik Boström381d1092020-05-12 18:49:07 +02001436 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1437 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1438 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001439 // The encoder will cache up to one frame for a short duration. Adding two
1440 // frames means that the first frame will be dropped and the second frame will
1441 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001442 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001443 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001444
Henrik Boström381d1092020-05-12 18:49:07 +02001445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001446 DataRate::BitsPerSec(kTargetBitrateBps),
1447 DataRate::BitsPerSec(kTargetBitrateBps),
1448 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001449 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001450 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1451 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001452 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001453}
1454
mflodmancc3d4422017-08-03 08:27:51 -07001455TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001457 DataRate::BitsPerSec(kTargetBitrateBps),
1458 DataRate::BitsPerSec(kTargetBitrateBps),
1459 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001460 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001462
1463 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001464 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001465
perkja49cbd32016-09-16 07:53:41 -07001466 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001467 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001468 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001469}
1470
mflodmancc3d4422017-08-03 08:27:51 -07001471TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001473 DataRate::BitsPerSec(kTargetBitrateBps),
1474 DataRate::BitsPerSec(kTargetBitrateBps),
1475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001476
perkja49cbd32016-09-16 07:53:41 -07001477 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001478 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001479
mflodmancc3d4422017-08-03 08:27:51 -07001480 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001481 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001482 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001483 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1484 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001485}
1486
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001487class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1488 public:
1489 VideoStreamEncoderBlockedTest() {}
1490
1491 TaskQueueFactory* GetTaskQueueFactory() override {
1492 return task_queue_factory_.get();
1493 }
1494
1495 private:
1496 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1497 CreateDefaultTaskQueueFactory();
1498};
1499
1500TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001502 DataRate::BitsPerSec(kTargetBitrateBps),
1503 DataRate::BitsPerSec(kTargetBitrateBps),
1504 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001505
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001506 int dropped_count = 0;
1507 stats_proxy_->SetDroppedFrameCallback(
1508 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1509 ++dropped_count;
1510 });
1511
perkj26091b12016-09-01 01:17:40 -07001512 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001513 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001514 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001515 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1516 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001517 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1518 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001519 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001520 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001521
mflodmancc3d4422017-08-03 08:27:51 -07001522 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001523
1524 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001525}
1526
Noah Richards51db4212019-06-12 06:59:12 -07001527TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001528 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001529 DataRate::BitsPerSec(kTargetBitrateBps),
1530 DataRate::BitsPerSec(kTargetBitrateBps),
1531 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001532
1533 rtc::Event frame_destroyed_event;
1534 video_source_.IncomingCapturedFrame(
1535 CreateFakeNativeFrame(1, &frame_destroyed_event));
1536 ExpectDroppedFrame();
1537 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1538 video_stream_encoder_->Stop();
1539}
1540
1541TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1542 // Use the cropping factory.
1543 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001544 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001545 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1546 kMaxPayloadLength);
1547 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1548
1549 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001551 DataRate::BitsPerSec(kTargetBitrateBps),
1552 DataRate::BitsPerSec(kTargetBitrateBps),
1553 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001554 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1555 WaitForEncodedFrame(1);
1556 // The encoder will have been configured once.
1557 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1558 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1559 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1560
1561 // Now send in a fake frame that needs to be cropped as the width/height
1562 // aren't divisible by 4 (see CreateEncoderStreams above).
1563 rtc::Event frame_destroyed_event;
1564 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1565 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1566 ExpectDroppedFrame();
1567 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1568 video_stream_encoder_->Stop();
1569}
1570
Evan Shrubsole895556e2020-10-05 09:15:13 +02001571TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1573 DataRate::BitsPerSec(kTargetBitrateBps),
1574 DataRate::BitsPerSec(kTargetBitrateBps),
1575 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1576
1577 video_source_.IncomingCapturedFrame(
1578 CreateNV12Frame(1, codec_width_, codec_height_));
1579 WaitForEncodedFrame(1);
1580 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1581 fake_encoder_.GetLastInputPixelFormat());
1582 video_stream_encoder_->Stop();
1583}
1584
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001585TEST_F(VideoStreamEncoderTest,
1586 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1587 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1588 DataRate::BitsPerSec(kTargetBitrateBps),
1589 DataRate::BitsPerSec(kTargetBitrateBps),
1590 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1591
1592 fake_encoder_.SetPreferredPixelFormats({});
1593
1594 rtc::Event frame_destroyed_event;
1595 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1596 1, &frame_destroyed_event, codec_width_, codec_height_));
1597 WaitForEncodedFrame(1);
1598 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1599 fake_encoder_.GetLastInputPixelFormat());
1600 video_stream_encoder_->Stop();
1601}
1602
1603TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1605 DataRate::BitsPerSec(kTargetBitrateBps),
1606 DataRate::BitsPerSec(kTargetBitrateBps),
1607 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1608
1609 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1610
1611 rtc::Event frame_destroyed_event;
1612 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1613 1, &frame_destroyed_event, codec_width_, codec_height_));
1614 WaitForEncodedFrame(1);
1615 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1616 fake_encoder_.GetLastInputPixelFormat());
1617 video_stream_encoder_->Stop();
1618}
1619
1620TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1622 DataRate::BitsPerSec(kTargetBitrateBps),
1623 DataRate::BitsPerSec(kTargetBitrateBps),
1624 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1625
1626 // Fake NV12 native frame does not allow mapping to I444.
1627 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1628
1629 rtc::Event frame_destroyed_event;
1630 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1631 1, &frame_destroyed_event, codec_width_, codec_height_));
1632 WaitForEncodedFrame(1);
1633 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1634 fake_encoder_.GetLastInputPixelFormat());
1635 video_stream_encoder_->Stop();
1636}
1637
Evan Shrubsole895556e2020-10-05 09:15:13 +02001638TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1640 DataRate::BitsPerSec(kTargetBitrateBps),
1641 DataRate::BitsPerSec(kTargetBitrateBps),
1642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1643
1644 rtc::Event frame_destroyed_event;
1645 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1646 1, &frame_destroyed_event, codec_width_, codec_height_));
1647 WaitForEncodedFrame(1);
1648 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1649 fake_encoder_.GetLastInputPixelFormat());
1650 video_stream_encoder_->Stop();
1651}
1652
Ying Wang9b881ab2020-02-07 14:29:32 +01001653TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001654 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001655 DataRate::BitsPerSec(kTargetBitrateBps),
1656 DataRate::BitsPerSec(kTargetBitrateBps),
1657 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001658 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1659 WaitForEncodedFrame(1);
1660
Henrik Boström381d1092020-05-12 18:49:07 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001662 DataRate::BitsPerSec(kTargetBitrateBps),
1663 DataRate::BitsPerSec(kTargetBitrateBps),
1664 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001665 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1666 // frames. Adding two frames means that the first frame will be dropped and
1667 // the second frame will be sent to the encoder.
1668 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1669 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1670 WaitForEncodedFrame(3);
1671 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1672 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1673 WaitForEncodedFrame(5);
1674 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1675 video_stream_encoder_->Stop();
1676}
1677
mflodmancc3d4422017-08-03 08:27:51 -07001678TEST_F(VideoStreamEncoderTest,
1679 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001680 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001681 DataRate::BitsPerSec(kTargetBitrateBps),
1682 DataRate::BitsPerSec(kTargetBitrateBps),
1683 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001684 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001685
1686 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001687 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001688 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001689 // The encoder will have been configured once when the first frame is
1690 // received.
1691 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001692
1693 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001694 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001695 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001696 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001697 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001698
1699 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001700 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001701 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001702 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001703 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001704
mflodmancc3d4422017-08-03 08:27:51 -07001705 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001706}
1707
mflodmancc3d4422017-08-03 08:27:51 -07001708TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001710 DataRate::BitsPerSec(kTargetBitrateBps),
1711 DataRate::BitsPerSec(kTargetBitrateBps),
1712 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001713
1714 // Capture a frame and wait for it to synchronize with the encoder thread.
1715 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001716 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001717 // The encoder will have been configured once.
1718 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001719 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1720 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1721
1722 codec_width_ *= 2;
1723 codec_height_ *= 2;
1724 // Capture a frame with a higher resolution and wait for it to synchronize
1725 // with the encoder thread.
1726 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001727 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001728 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1729 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001730 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001731
mflodmancc3d4422017-08-03 08:27:51 -07001732 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001733}
1734
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001735TEST_F(VideoStreamEncoderTest,
1736 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001738 DataRate::BitsPerSec(kTargetBitrateBps),
1739 DataRate::BitsPerSec(kTargetBitrateBps),
1740 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001741
1742 // Capture a frame and wait for it to synchronize with the encoder thread.
1743 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1744 WaitForEncodedFrame(1);
1745
1746 VideoEncoderConfig video_encoder_config;
1747 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1748 // Changing the max payload data length recreates encoder.
1749 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1750 kMaxPayloadLength / 2);
1751
1752 // Capture a frame and wait for it to synchronize with the encoder thread.
1753 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1754 WaitForEncodedFrame(2);
1755 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1756
1757 video_stream_encoder_->Stop();
1758}
1759
Sergey Silkin5ee69672019-07-02 14:18:34 +02001760TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001762 DataRate::BitsPerSec(kTargetBitrateBps),
1763 DataRate::BitsPerSec(kTargetBitrateBps),
1764 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001765
1766 VideoEncoderConfig video_encoder_config;
1767 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1768 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1769 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1770 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1771 kMaxPayloadLength);
1772
1773 // Capture a frame and wait for it to synchronize with the encoder thread.
1774 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1775 WaitForEncodedFrame(1);
1776 // The encoder will have been configured once when the first frame is
1777 // received.
1778 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1779 EXPECT_EQ(kTargetBitrateBps,
1780 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1781 EXPECT_EQ(kStartBitrateBps,
1782 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1783
Sergey Silkin6456e352019-07-08 17:56:40 +02001784 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1785 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001786 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1787 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1788 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1789 kMaxPayloadLength);
1790
1791 // Capture a frame and wait for it to synchronize with the encoder thread.
1792 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1793 WaitForEncodedFrame(2);
1794 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1795 // Bitrate limits have changed - rate allocator should be reconfigured,
1796 // encoder should not be reconfigured.
1797 EXPECT_EQ(kTargetBitrateBps * 2,
1798 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1799 EXPECT_EQ(kStartBitrateBps * 2,
1800 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1801 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1802
1803 video_stream_encoder_->Stop();
1804}
1805
Sergey Silkin6456e352019-07-08 17:56:40 +02001806TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001807 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001809 DataRate::BitsPerSec(kTargetBitrateBps),
1810 DataRate::BitsPerSec(kTargetBitrateBps),
1811 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001812
Sergey Silkincd02eba2020-01-20 14:48:40 +01001813 const uint32_t kMinEncBitrateKbps = 100;
1814 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001815 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001816 /*frame_size_pixels=*/codec_width_ * codec_height_,
1817 /*min_start_bitrate_bps=*/0,
1818 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1819 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001820 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1821
Sergey Silkincd02eba2020-01-20 14:48:40 +01001822 VideoEncoderConfig video_encoder_config;
1823 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1824 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1825 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1826 (kMinEncBitrateKbps + 1) * 1000;
1827 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1828 kMaxPayloadLength);
1829
1830 // When both encoder and app provide bitrate limits, the intersection of
1831 // provided sets should be used.
1832 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1833 WaitForEncodedFrame(1);
1834 EXPECT_EQ(kMaxEncBitrateKbps,
1835 bitrate_allocator_factory_.codec_config().maxBitrate);
1836 EXPECT_EQ(kMinEncBitrateKbps + 1,
1837 bitrate_allocator_factory_.codec_config().minBitrate);
1838
1839 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1840 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1841 (kMinEncBitrateKbps - 1) * 1000;
1842 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1843 kMaxPayloadLength);
1844 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001845 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001846 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001847 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001848 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001849 bitrate_allocator_factory_.codec_config().minBitrate);
1850
Sergey Silkincd02eba2020-01-20 14:48:40 +01001851 video_stream_encoder_->Stop();
1852}
1853
1854TEST_F(VideoStreamEncoderTest,
1855 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001857 DataRate::BitsPerSec(kTargetBitrateBps),
1858 DataRate::BitsPerSec(kTargetBitrateBps),
1859 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001860
1861 const uint32_t kMinAppBitrateKbps = 100;
1862 const uint32_t kMaxAppBitrateKbps = 200;
1863 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1864 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1865 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1866 /*frame_size_pixels=*/codec_width_ * codec_height_,
1867 /*min_start_bitrate_bps=*/0,
1868 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1869 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1870 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1871
1872 VideoEncoderConfig video_encoder_config;
1873 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1874 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1875 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1876 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001877 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1878 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001879
Sergey Silkincd02eba2020-01-20 14:48:40 +01001880 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1881 WaitForEncodedFrame(1);
1882 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001884 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001885 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001886
1887 video_stream_encoder_->Stop();
1888}
1889
1890TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001891 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001893 DataRate::BitsPerSec(kTargetBitrateBps),
1894 DataRate::BitsPerSec(kTargetBitrateBps),
1895 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001896
1897 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001898 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001899 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001900 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001901 fake_encoder_.SetResolutionBitrateLimits(
1902 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1903
1904 VideoEncoderConfig video_encoder_config;
1905 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1906 video_encoder_config.max_bitrate_bps = 0;
1907 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1908 kMaxPayloadLength);
1909
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001910 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001911 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1912 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001913 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1914 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001915 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1916 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1917
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001918 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001919 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1920 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001921 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1922 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001923 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1924 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1925
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001926 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001927 // encoder for 360p should be used.
1928 video_source_.IncomingCapturedFrame(
1929 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1930 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001931 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1932 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001933 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1934 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1935
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001936 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001937 // ignored.
1938 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1939 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001940 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1941 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001942 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1943 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001944 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1945 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001946 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1948
1949 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1950 // for 270p should be used.
1951 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1952 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001953 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1954 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001955 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1956 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1957
1958 video_stream_encoder_->Stop();
1959}
1960
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001961TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001962 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001963 DataRate::BitsPerSec(kTargetBitrateBps),
1964 DataRate::BitsPerSec(kTargetBitrateBps),
1965 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001966
1967 VideoEncoderConfig video_encoder_config;
1968 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1969 video_encoder_config.max_bitrate_bps = 0;
1970 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1971 kMaxPayloadLength);
1972
1973 // Encode 720p frame to get the default encoder target bitrate.
1974 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1975 WaitForEncodedFrame(1);
1976 const uint32_t kDefaultTargetBitrateFor720pKbps =
1977 bitrate_allocator_factory_.codec_config()
1978 .simulcastStream[0]
1979 .targetBitrate;
1980
1981 // Set the max recommended encoder bitrate to something lower than the default
1982 // target bitrate.
1983 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1984 1280 * 720, 10 * 1000, 10 * 1000,
1985 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1986 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1987
1988 // Change resolution to trigger encoder reinitialization.
1989 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1990 WaitForEncodedFrame(2);
1991 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1992 WaitForEncodedFrame(3);
1993
1994 // Ensure the target bitrate is capped by the max bitrate.
1995 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1996 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1997 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1998 .simulcastStream[0]
1999 .targetBitrate *
2000 1000,
2001 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2002
2003 video_stream_encoder_->Stop();
2004}
2005
mflodmancc3d4422017-08-03 08:27:51 -07002006TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002007 EXPECT_TRUE(video_source_.has_sinks());
2008 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002009 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002010 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002011 EXPECT_FALSE(video_source_.has_sinks());
2012 EXPECT_TRUE(new_video_source.has_sinks());
2013
mflodmancc3d4422017-08-03 08:27:51 -07002014 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002015}
2016
mflodmancc3d4422017-08-03 08:27:51 -07002017TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002018 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002019 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002020 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002021 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002022}
2023
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002024class ResolutionAlignmentTest
2025 : public VideoStreamEncoderTest,
2026 public ::testing::WithParamInterface<
2027 ::testing::tuple<int, std::vector<double>>> {
2028 public:
2029 ResolutionAlignmentTest()
2030 : requested_alignment_(::testing::get<0>(GetParam())),
2031 scale_factors_(::testing::get<1>(GetParam())) {}
2032
2033 protected:
2034 const int requested_alignment_;
2035 const std::vector<double> scale_factors_;
2036};
2037
2038INSTANTIATE_TEST_SUITE_P(
2039 AlignmentAndScaleFactors,
2040 ResolutionAlignmentTest,
2041 ::testing::Combine(
2042 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2043 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2044 std::vector<double>{-1.0, -1.0},
2045 std::vector<double>{-1.0, -1.0, -1.0},
2046 std::vector<double>{4.0, 2.0, 1.0},
2047 std::vector<double>{9999.0, -1.0, 1.0},
2048 std::vector<double>{3.99, 2.01, 1.0},
2049 std::vector<double>{4.9, 1.7, 1.25},
2050 std::vector<double>{10.0, 4.0, 3.0},
2051 std::vector<double>{1.75, 3.5},
2052 std::vector<double>{1.5, 2.5},
2053 std::vector<double>{1.3, 1.0})));
2054
2055TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2056 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002057 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002058 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2059 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2060
2061 // Fill config with the scaling factor by which to reduce encoding size.
2062 const int num_streams = scale_factors_.size();
2063 VideoEncoderConfig config;
2064 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2065 for (int i = 0; i < num_streams; ++i) {
2066 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2067 }
2068 config.video_stream_factory =
2069 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2070 "VP8", /*max qp*/ 56, /*screencast*/ false,
2071 /*screenshare enabled*/ false);
2072 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2073
Henrik Boström381d1092020-05-12 18:49:07 +02002074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002075 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2076 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2077 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2078 // Wait for all layers before triggering event.
2079 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002080
2081 // On the 1st frame, we should have initialized the encoder and
2082 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002083 int64_t timestamp_ms = kFrameIntervalMs;
2084 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2085 WaitForEncodedFrame(timestamp_ms);
2086 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002087
2088 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2089 // (It's up the to the encoder to potentially drop the previous frame,
2090 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002091 timestamp_ms += kFrameIntervalMs;
2092 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2093 WaitForEncodedFrame(timestamp_ms);
2094 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2095
2096 VideoCodec codec = fake_encoder_.video_codec();
2097 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2098 // Frame size should be a multiple of the requested alignment.
2099 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2100 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2101 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2102 // Aspect ratio should match.
2103 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2104 codec.height * codec.simulcastStream[i].width);
2105 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002106
2107 video_stream_encoder_->Stop();
2108}
2109
Jonathan Yubc771b72017-12-08 17:04:29 -08002110TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2111 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002112 const int kWidth = 1280;
2113 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002114
2115 // We rely on the automatic resolution adaptation, but we handle framerate
2116 // adaptation manually by mocking the stats proxy.
2117 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002118
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002119 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002120 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002121 DataRate::BitsPerSec(kTargetBitrateBps),
2122 DataRate::BitsPerSec(kTargetBitrateBps),
2123 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002124 video_stream_encoder_->SetSource(&video_source_,
2125 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002126 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002127 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002128 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002129 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2130
Jonathan Yubc771b72017-12-08 17:04:29 -08002131 // Adapt down as far as possible.
2132 rtc::VideoSinkWants last_wants;
2133 int64_t t = 1;
2134 int loop_count = 0;
2135 do {
2136 ++loop_count;
2137 last_wants = video_source_.sink_wants();
2138
2139 // Simulate the framerate we've been asked to adapt to.
2140 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2141 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2142 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2143 mock_stats.input_frame_rate = fps;
2144 stats_proxy_->SetMockStats(mock_stats);
2145
2146 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2147 sink_.WaitForEncodedFrame(t);
2148 t += frame_interval_ms;
2149
mflodmancc3d4422017-08-03 08:27:51 -07002150 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002151 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002152 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002153 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2154 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002155 } while (video_source_.sink_wants().max_pixel_count <
2156 last_wants.max_pixel_count ||
2157 video_source_.sink_wants().max_framerate_fps <
2158 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002159
Jonathan Yubc771b72017-12-08 17:04:29 -08002160 // Verify that we've adapted all the way down.
2161 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002162 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002163 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2164 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002165 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002166 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2167 *video_source_.last_sent_height());
2168 EXPECT_EQ(kMinBalancedFramerateFps,
2169 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002170
Jonathan Yubc771b72017-12-08 17:04:29 -08002171 // Adapt back up the same number of times we adapted down.
2172 for (int i = 0; i < loop_count - 1; ++i) {
2173 last_wants = video_source_.sink_wants();
2174
2175 // Simulate the framerate we've been asked to adapt to.
2176 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2177 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2178 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2179 mock_stats.input_frame_rate = fps;
2180 stats_proxy_->SetMockStats(mock_stats);
2181
2182 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2183 sink_.WaitForEncodedFrame(t);
2184 t += frame_interval_ms;
2185
Henrik Boström91aa7322020-04-28 12:24:33 +02002186 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002187 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002188 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002189 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2190 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002191 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2192 last_wants.max_pixel_count ||
2193 video_source_.sink_wants().max_framerate_fps >
2194 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002195 }
2196
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002197 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002198 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002199 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002200 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2201 EXPECT_EQ((loop_count - 1) * 2,
2202 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002203
mflodmancc3d4422017-08-03 08:27:51 -07002204 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002205}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002206
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002207TEST_F(VideoStreamEncoderTest,
2208 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2209 video_stream_encoder_->OnBitrateUpdated(
2210 DataRate::BitsPerSec(kTargetBitrateBps),
2211 DataRate::BitsPerSec(kTargetBitrateBps),
2212 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002213 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002214
2215 const int kFrameWidth = 1280;
2216 const int kFrameHeight = 720;
2217
2218 int64_t ntp_time = kFrameIntervalMs;
2219
2220 // Force an input frame rate to be available, or the adaptation call won't
2221 // know what framerate to adapt form.
2222 const int kInputFps = 30;
2223 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2224 stats.input_frame_rate = kInputFps;
2225 stats_proxy_->SetMockStats(stats);
2226
2227 video_source_.set_adaptation_enabled(true);
2228 video_stream_encoder_->SetSource(
2229 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002230 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002231 video_source_.IncomingCapturedFrame(
2232 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2233 sink_.WaitForEncodedFrame(ntp_time);
2234 ntp_time += kFrameIntervalMs;
2235
2236 // Trigger CPU overuse.
2237 video_stream_encoder_->TriggerCpuOveruse();
2238 video_source_.IncomingCapturedFrame(
2239 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2240 sink_.WaitForEncodedFrame(ntp_time);
2241 ntp_time += kFrameIntervalMs;
2242
2243 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2244 EXPECT_EQ(std::numeric_limits<int>::max(),
2245 video_source_.sink_wants().max_pixel_count);
2246 // Some framerate constraint should be set.
2247 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2248 EXPECT_LT(restricted_fps, kInputFps);
2249 video_source_.IncomingCapturedFrame(
2250 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2251 sink_.WaitForEncodedFrame(ntp_time);
2252 ntp_time += 100;
2253
Henrik Boström2671dac2020-05-19 16:29:09 +02002254 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002255 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2256 // Give the encoder queue time to process the change in degradation preference
2257 // by waiting for an encoded frame.
2258 video_source_.IncomingCapturedFrame(
2259 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2260 sink_.WaitForEncodedFrame(ntp_time);
2261 ntp_time += kFrameIntervalMs;
2262
2263 video_stream_encoder_->TriggerQualityLow();
2264 video_source_.IncomingCapturedFrame(
2265 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2266 sink_.WaitForEncodedFrame(ntp_time);
2267 ntp_time += kFrameIntervalMs;
2268
2269 // Some resolution constraint should be set.
2270 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2271 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2272 kFrameWidth * kFrameHeight);
2273 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2274
2275 int pixel_count = video_source_.sink_wants().max_pixel_count;
2276 // Triggering a CPU underuse should not change the sink wants since it has
2277 // not been overused for resolution since we changed degradation preference.
2278 video_stream_encoder_->TriggerCpuUnderuse();
2279 video_source_.IncomingCapturedFrame(
2280 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2281 sink_.WaitForEncodedFrame(ntp_time);
2282 ntp_time += kFrameIntervalMs;
2283 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2284 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2285
Evan Shrubsole64469032020-06-11 10:45:29 +02002286 // Change the degradation preference back. CPU underuse should not adapt since
2287 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002288 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002289 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2290 video_source_.IncomingCapturedFrame(
2291 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2292 sink_.WaitForEncodedFrame(ntp_time);
2293 ntp_time += 100;
2294 // Resolution adaptations is gone after changing degradation preference.
2295 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2296 EXPECT_EQ(std::numeric_limits<int>::max(),
2297 video_source_.sink_wants().max_pixel_count);
2298 // The fps adaptation from above is now back.
2299 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2300
2301 // Trigger CPU underuse.
2302 video_stream_encoder_->TriggerCpuUnderuse();
2303 video_source_.IncomingCapturedFrame(
2304 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2305 sink_.WaitForEncodedFrame(ntp_time);
2306 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002307 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2308
2309 // Trigger QP underuse, fps should return to normal.
2310 video_stream_encoder_->TriggerQualityHigh();
2311 video_source_.IncomingCapturedFrame(
2312 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2313 sink_.WaitForEncodedFrame(ntp_time);
2314 ntp_time += kFrameIntervalMs;
2315 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002316
2317 video_stream_encoder_->Stop();
2318}
2319
mflodmancc3d4422017-08-03 08:27:51 -07002320TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002322 DataRate::BitsPerSec(kTargetBitrateBps),
2323 DataRate::BitsPerSec(kTargetBitrateBps),
2324 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002325 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002326
sprangc5d62e22017-04-02 23:53:04 -07002327 const int kFrameWidth = 1280;
2328 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002329
Åsa Persson8c1bf952018-09-13 10:42:19 +02002330 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002331
kthelgason5e13d412016-12-01 03:59:51 -08002332 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002333 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002334 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002335 frame_timestamp += kFrameIntervalMs;
2336
perkj803d97f2016-11-01 11:45:46 -07002337 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002338 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002339 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002340 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002341 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002342 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002343
asapersson0944a802017-04-07 00:57:58 -07002344 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002345 // wanted resolution.
2346 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2347 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2348 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002349 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002350
2351 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002352 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002353 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002354 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002355 // Give the encoder queue time to process the change in degradation preference
2356 // by waiting for an encoded frame.
2357 new_video_source.IncomingCapturedFrame(
2358 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2359 sink_.WaitForEncodedFrame(frame_timestamp);
2360 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002361 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002362 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002363
sprangc5d62e22017-04-02 23:53:04 -07002364 // Force an input frame rate to be available, or the adaptation call won't
2365 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002366 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002367 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002368 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002369 stats_proxy_->SetMockStats(stats);
2370
mflodmancc3d4422017-08-03 08:27:51 -07002371 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002372 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002373 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002374 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002375 frame_timestamp += kFrameIntervalMs;
2376
2377 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002378 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002379 EXPECT_EQ(std::numeric_limits<int>::max(),
2380 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002381 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002382
asapersson02465b82017-04-10 01:12:52 -07002383 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002384 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2385 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002386 // Give the encoder queue time to process the change in degradation preference
2387 // by waiting for an encoded frame.
2388 new_video_source.IncomingCapturedFrame(
2389 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2390 sink_.WaitForEncodedFrame(frame_timestamp);
2391 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002392 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002393
mflodmancc3d4422017-08-03 08:27:51 -07002394 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002395 new_video_source.IncomingCapturedFrame(
2396 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002397 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002398 frame_timestamp += kFrameIntervalMs;
2399
2400 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002401 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002402
2403 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002404 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002405 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002406 // Give the encoder queue time to process the change in degradation preference
2407 // by waiting for an encoded frame.
2408 new_video_source.IncomingCapturedFrame(
2409 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2410 sink_.WaitForEncodedFrame(frame_timestamp);
2411 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002412 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2413 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002414 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002415 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002416
2417 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002418 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002419 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002420 // Give the encoder queue time to process the change in degradation preference
2421 // by waiting for an encoded frame.
2422 new_video_source.IncomingCapturedFrame(
2423 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2424 sink_.WaitForEncodedFrame(frame_timestamp);
2425 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002426 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2427 EXPECT_EQ(std::numeric_limits<int>::max(),
2428 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002429 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002430
mflodmancc3d4422017-08-03 08:27:51 -07002431 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002432}
2433
mflodmancc3d4422017-08-03 08:27:51 -07002434TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002436 DataRate::BitsPerSec(kTargetBitrateBps),
2437 DataRate::BitsPerSec(kTargetBitrateBps),
2438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002439
asaperssonfab67072017-04-04 05:51:49 -07002440 const int kWidth = 1280;
2441 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002442 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002443 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002444 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2445 EXPECT_FALSE(stats.bw_limited_resolution);
2446 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2447
2448 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002449 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002450 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002451 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002452
2453 stats = stats_proxy_->GetStats();
2454 EXPECT_TRUE(stats.bw_limited_resolution);
2455 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2456
2457 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002458 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002459 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002460 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002461
2462 stats = stats_proxy_->GetStats();
2463 EXPECT_FALSE(stats.bw_limited_resolution);
2464 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2465 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2466
mflodmancc3d4422017-08-03 08:27:51 -07002467 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002468}
2469
mflodmancc3d4422017-08-03 08:27:51 -07002470TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002472 DataRate::BitsPerSec(kTargetBitrateBps),
2473 DataRate::BitsPerSec(kTargetBitrateBps),
2474 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002475
2476 const int kWidth = 1280;
2477 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002478 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002479 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002480 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2481 EXPECT_FALSE(stats.cpu_limited_resolution);
2482 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2483
2484 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002485 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002486 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002487 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002488
2489 stats = stats_proxy_->GetStats();
2490 EXPECT_TRUE(stats.cpu_limited_resolution);
2491 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2492
2493 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002494 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002495 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002496 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002497
2498 stats = stats_proxy_->GetStats();
2499 EXPECT_FALSE(stats.cpu_limited_resolution);
2500 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002501 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002502
mflodmancc3d4422017-08-03 08:27:51 -07002503 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002504}
2505
mflodmancc3d4422017-08-03 08:27:51 -07002506TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002508 DataRate::BitsPerSec(kTargetBitrateBps),
2509 DataRate::BitsPerSec(kTargetBitrateBps),
2510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002511
asaperssonfab67072017-04-04 05:51:49 -07002512 const int kWidth = 1280;
2513 const int kHeight = 720;
2514 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002515 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002516 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002517 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002518 EXPECT_FALSE(stats.cpu_limited_resolution);
2519 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2520
asaperssonfab67072017-04-04 05:51:49 -07002521 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002522 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002523 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002524 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002525 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002526 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002527 EXPECT_TRUE(stats.cpu_limited_resolution);
2528 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2529
2530 // Set new source with adaptation still enabled.
2531 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002532 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002533 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002534
asaperssonfab67072017-04-04 05:51:49 -07002535 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002536 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002537 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002538 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002539 EXPECT_TRUE(stats.cpu_limited_resolution);
2540 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2541
2542 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002543 video_stream_encoder_->SetSource(&new_video_source,
2544 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002545
asaperssonfab67072017-04-04 05:51:49 -07002546 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002547 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002548 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002549 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002550 EXPECT_FALSE(stats.cpu_limited_resolution);
2551 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2552
2553 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002554 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002555 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002556
asaperssonfab67072017-04-04 05:51:49 -07002557 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002558 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002559 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002560 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002561 EXPECT_TRUE(stats.cpu_limited_resolution);
2562 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2563
asaperssonfab67072017-04-04 05:51:49 -07002564 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002565 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002566 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002567 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002568 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002569 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002570 EXPECT_FALSE(stats.cpu_limited_resolution);
2571 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002572 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002573
mflodmancc3d4422017-08-03 08:27:51 -07002574 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002575}
2576
mflodmancc3d4422017-08-03 08:27:51 -07002577TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002579 DataRate::BitsPerSec(kTargetBitrateBps),
2580 DataRate::BitsPerSec(kTargetBitrateBps),
2581 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002582
asaperssonfab67072017-04-04 05:51:49 -07002583 const int kWidth = 1280;
2584 const int kHeight = 720;
2585 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002586 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002587 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002588 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002589 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002590 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002591
2592 // Set new source with adaptation still enabled.
2593 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002594 video_stream_encoder_->SetSource(&new_video_source,
2595 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002596
asaperssonfab67072017-04-04 05:51:49 -07002597 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002598 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002599 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002600 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002601 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002602 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002603
asaperssonfab67072017-04-04 05:51:49 -07002604 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002605 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002606 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002607 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002608 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002609 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002610 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002611 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002612
asaperssonfab67072017-04-04 05:51:49 -07002613 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002614 video_stream_encoder_->SetSource(&new_video_source,
2615 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002616
asaperssonfab67072017-04-04 05:51:49 -07002617 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002618 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002619 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002620 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002621 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002622 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002623
asapersson02465b82017-04-10 01:12:52 -07002624 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002625 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002626 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002627
asaperssonfab67072017-04-04 05:51:49 -07002628 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002629 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002630 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002631 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002632 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002633 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2634 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002635
mflodmancc3d4422017-08-03 08:27:51 -07002636 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002637}
2638
mflodmancc3d4422017-08-03 08:27:51 -07002639TEST_F(VideoStreamEncoderTest,
2640 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002642 DataRate::BitsPerSec(kTargetBitrateBps),
2643 DataRate::BitsPerSec(kTargetBitrateBps),
2644 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002645
2646 const int kWidth = 1280;
2647 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002648 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002649 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002650 video_source_.IncomingCapturedFrame(
2651 CreateFrame(timestamp_ms, kWidth, kHeight));
2652 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002653 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2655 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2656
2657 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002658 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002659 timestamp_ms += kFrameIntervalMs;
2660 video_source_.IncomingCapturedFrame(
2661 CreateFrame(timestamp_ms, kWidth, kHeight));
2662 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002663 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2664 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2665 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2666
2667 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002668 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002669 timestamp_ms += kFrameIntervalMs;
2670 video_source_.IncomingCapturedFrame(
2671 CreateFrame(timestamp_ms, kWidth, kHeight));
2672 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002673 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2674 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2675 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2676
Niels Möller4db138e2018-04-19 09:04:13 +02002677 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002678 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002679
2680 VideoEncoderConfig video_encoder_config;
2681 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2682 // Make format different, to force recreation of encoder.
2683 video_encoder_config.video_format.parameters["foo"] = "foo";
2684 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002685 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002686 timestamp_ms += kFrameIntervalMs;
2687 video_source_.IncomingCapturedFrame(
2688 CreateFrame(timestamp_ms, kWidth, kHeight));
2689 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002690 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2691 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2692 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2693
mflodmancc3d4422017-08-03 08:27:51 -07002694 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002695}
2696
mflodmancc3d4422017-08-03 08:27:51 -07002697TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002698 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002700 DataRate::BitsPerSec(kTargetBitrateBps),
2701 DataRate::BitsPerSec(kTargetBitrateBps),
2702 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2703
2704 const int kWidth = 1280;
2705 const int kHeight = 720;
2706 int sequence = 1;
2707
2708 // Enable BALANCED preference, no initial limitation.
2709 test::FrameForwarder source;
2710 video_stream_encoder_->SetSource(&source,
2711 webrtc::DegradationPreference::BALANCED);
2712 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2713 WaitForEncodedFrame(sequence++);
2714 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2715 EXPECT_FALSE(stats.cpu_limited_resolution);
2716 EXPECT_FALSE(stats.cpu_limited_framerate);
2717 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2718
2719 // Trigger CPU overuse, should now adapt down.
2720 video_stream_encoder_->TriggerCpuOveruse();
2721 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2722 WaitForEncodedFrame(sequence++);
2723 stats = stats_proxy_->GetStats();
2724 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2725
2726 // Set new degradation preference should clear restrictions since we changed
2727 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002728 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002729 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2730 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2731 WaitForEncodedFrame(sequence++);
2732 stats = stats_proxy_->GetStats();
2733 EXPECT_FALSE(stats.cpu_limited_resolution);
2734 EXPECT_FALSE(stats.cpu_limited_framerate);
2735 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2736
2737 // Force an input frame rate to be available, or the adaptation call won't
2738 // know what framerate to adapt from.
2739 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2740 mock_stats.input_frame_rate = 30;
2741 stats_proxy_->SetMockStats(mock_stats);
2742 video_stream_encoder_->TriggerCpuOveruse();
2743 stats_proxy_->ResetMockStats();
2744 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2745 WaitForEncodedFrame(sequence++);
2746
2747 // We have now adapted once.
2748 stats = stats_proxy_->GetStats();
2749 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2750
2751 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002752 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2753 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002754 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2755 WaitForEncodedFrame(sequence++);
2756 stats = stats_proxy_->GetStats();
2757 EXPECT_FALSE(stats.cpu_limited_resolution);
2758 EXPECT_FALSE(stats.cpu_limited_framerate);
2759 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2760
2761 video_stream_encoder_->Stop();
2762}
2763
2764TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002765 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002767 DataRate::BitsPerSec(kTargetBitrateBps),
2768 DataRate::BitsPerSec(kTargetBitrateBps),
2769 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002770
asapersson0944a802017-04-07 00:57:58 -07002771 const int kWidth = 1280;
2772 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002773 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002774
asaperssonfab67072017-04-04 05:51:49 -07002775 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002776 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002777 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002778 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002779 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002780 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2781
asapersson02465b82017-04-10 01:12:52 -07002782 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002783 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002784 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002785 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002786 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002787 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002788 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002789 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2790
2791 // Set new source with adaptation still enabled.
2792 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002793 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002794 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002795
2796 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002797 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002798 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002799 stats = stats_proxy_->GetStats();
2800 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002801 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002802 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2803
sprangc5d62e22017-04-02 23:53:04 -07002804 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002805 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002806 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002807 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002808 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002809 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002810 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002811 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002812 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002813 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002814 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2815
sprangc5d62e22017-04-02 23:53:04 -07002816 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002817 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002818 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2819 mock_stats.input_frame_rate = 30;
2820 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002821 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002822 stats_proxy_->ResetMockStats();
2823
2824 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002825 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002826 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002827
2828 // Framerate now adapted.
2829 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002830 EXPECT_FALSE(stats.cpu_limited_resolution);
2831 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002832 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2833
2834 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002835 video_stream_encoder_->SetSource(&new_video_source,
2836 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002837 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002838 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002839 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002840
2841 stats = stats_proxy_->GetStats();
2842 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002843 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002844 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2845
2846 // Try to trigger overuse. Should not succeed.
2847 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002848 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002849 stats_proxy_->ResetMockStats();
2850
2851 stats = stats_proxy_->GetStats();
2852 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002853 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002854 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2855
2856 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002858 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002859 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002860 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002861 stats = stats_proxy_->GetStats();
2862 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002863 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002864 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002865
2866 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002867 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002868 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002869 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002870 stats = stats_proxy_->GetStats();
2871 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002872 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002873 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2874
2875 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002876 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002877 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002878 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002879 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002880 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002881 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002882 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002883 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002884 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002885 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2886
2887 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002888 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002889 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002890 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002891 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002892 stats = stats_proxy_->GetStats();
2893 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002894 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002895 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002896 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002897
mflodmancc3d4422017-08-03 08:27:51 -07002898 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002899}
2900
mflodmancc3d4422017-08-03 08:27:51 -07002901TEST_F(VideoStreamEncoderTest,
2902 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002903 const int kWidth = 1280;
2904 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002905 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002906 DataRate::BitsPerSec(kTargetBitrateBps),
2907 DataRate::BitsPerSec(kTargetBitrateBps),
2908 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002909
asaperssonfab67072017-04-04 05:51:49 -07002910 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002911 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002912
asaperssonfab67072017-04-04 05:51:49 -07002913 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002914 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002915
asaperssonfab67072017-04-04 05:51:49 -07002916 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002917 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002918
asaperssonfab67072017-04-04 05:51:49 -07002919 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002920 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002921
kthelgason876222f2016-11-29 01:44:11 -08002922 // Expect a scale down.
2923 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002924 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002925
asapersson02465b82017-04-10 01:12:52 -07002926 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002927 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002929 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002930
asaperssonfab67072017-04-04 05:51:49 -07002931 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002932 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002933 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002934 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002935
asaperssonfab67072017-04-04 05:51:49 -07002936 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002937 EXPECT_EQ(std::numeric_limits<int>::max(),
2938 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002939
asaperssonfab67072017-04-04 05:51:49 -07002940 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002941 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002942 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002943 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002944
asapersson02465b82017-04-10 01:12:52 -07002945 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002946 EXPECT_EQ(std::numeric_limits<int>::max(),
2947 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002948
mflodmancc3d4422017-08-03 08:27:51 -07002949 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002950}
2951
mflodmancc3d4422017-08-03 08:27:51 -07002952TEST_F(VideoStreamEncoderTest,
2953 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002954 const int kWidth = 1280;
2955 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002957 DataRate::BitsPerSec(kTargetBitrateBps),
2958 DataRate::BitsPerSec(kTargetBitrateBps),
2959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002960
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002961 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002962 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002963 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002964 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002965
2966 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002967 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002968 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002969 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2970 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2971
2972 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002973 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002974 EXPECT_THAT(source.sink_wants(),
2975 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002976 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2977 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2978 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2979
2980 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002981 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002982 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2983 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2984 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2985
mflodmancc3d4422017-08-03 08:27:51 -07002986 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002987}
2988
mflodmancc3d4422017-08-03 08:27:51 -07002989TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002990 const int kWidth = 1280;
2991 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002993 DataRate::BitsPerSec(kTargetBitrateBps),
2994 DataRate::BitsPerSec(kTargetBitrateBps),
2995 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002996
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002997 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002998 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002999 video_stream_encoder_->SetSource(&source,
3000 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003001 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3002 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003003 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003004
3005 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003006 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003007 EXPECT_THAT(source.sink_wants(),
3008 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003009 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3010 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3011 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3012
3013 // Trigger adapt down for same input resolution, expect no change.
3014 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3015 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003016 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003017 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3018 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3019 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3020
3021 // Trigger adapt down for larger input resolution, expect no change.
3022 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3023 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003024 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003025 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3026 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3027 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3028
mflodmancc3d4422017-08-03 08:27:51 -07003029 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003030}
3031
mflodmancc3d4422017-08-03 08:27:51 -07003032TEST_F(VideoStreamEncoderTest,
3033 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003034 const int kWidth = 1280;
3035 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003037 DataRate::BitsPerSec(kTargetBitrateBps),
3038 DataRate::BitsPerSec(kTargetBitrateBps),
3039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003040
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003041 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003042 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003043 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003044 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003045
3046 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003047 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003048 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003049 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3050 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3051
3052 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003053 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003054 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003055 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3056 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3057
mflodmancc3d4422017-08-03 08:27:51 -07003058 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003059}
3060
mflodmancc3d4422017-08-03 08:27:51 -07003061TEST_F(VideoStreamEncoderTest,
3062 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003063 const int kWidth = 1280;
3064 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003065 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003066 DataRate::BitsPerSec(kTargetBitrateBps),
3067 DataRate::BitsPerSec(kTargetBitrateBps),
3068 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003069
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003070 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003071 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003072 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003073 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003074
3075 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003076 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003077 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003078 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003079 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3080
3081 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003082 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003083 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003084 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003085 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3086
mflodmancc3d4422017-08-03 08:27:51 -07003087 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003088}
3089
mflodmancc3d4422017-08-03 08:27:51 -07003090TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003091 const int kWidth = 1280;
3092 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003094 DataRate::BitsPerSec(kTargetBitrateBps),
3095 DataRate::BitsPerSec(kTargetBitrateBps),
3096 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003097
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003098 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003099 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003100 video_stream_encoder_->SetSource(&source,
3101 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003102
3103 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3104 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003105 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003106 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3107 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3108 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3109
3110 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003111 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003112 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003113 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3114 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3115 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3116
mflodmancc3d4422017-08-03 08:27:51 -07003117 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003118}
3119
mflodmancc3d4422017-08-03 08:27:51 -07003120TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003121 const int kWidth = 1280;
3122 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003124 DataRate::BitsPerSec(kTargetBitrateBps),
3125 DataRate::BitsPerSec(kTargetBitrateBps),
3126 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003127
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003128 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003129 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003130 video_stream_encoder_->SetSource(&source,
3131 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003132
3133 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3134 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003135 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003136 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3137 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3138 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3139
3140 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003141 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003142 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003143 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3144 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3145 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3146
mflodmancc3d4422017-08-03 08:27:51 -07003147 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003148}
3149
mflodmancc3d4422017-08-03 08:27:51 -07003150TEST_F(VideoStreamEncoderTest,
3151 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003152 const int kWidth = 1280;
3153 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003155 DataRate::BitsPerSec(kTargetBitrateBps),
3156 DataRate::BitsPerSec(kTargetBitrateBps),
3157 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003158
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003159 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003160 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003161 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003163 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003164
3165 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003166 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003167 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003168 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3169 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3170
3171 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003172 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003173 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003174 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003175 EXPECT_THAT(source.sink_wants(),
3176 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003177 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3178 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3179
3180 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003181 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003182 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003183 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3184 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3185 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3186
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003188}
3189
mflodmancc3d4422017-08-03 08:27:51 -07003190TEST_F(VideoStreamEncoderTest,
3191 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003192 const int kWidth = 1280;
3193 const int kHeight = 720;
3194 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003196 DataRate::BitsPerSec(kTargetBitrateBps),
3197 DataRate::BitsPerSec(kTargetBitrateBps),
3198 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003199
3200 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3201 stats.input_frame_rate = kInputFps;
3202 stats_proxy_->SetMockStats(stats);
3203
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003204 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003205 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3206 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003207 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003208
3209 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003210 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003211 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3212 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003213 EXPECT_THAT(video_source_.sink_wants(),
3214 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003215
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003216 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003217 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003218 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003219 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003220 // Give the encoder queue time to process the change in degradation preference
3221 // by waiting for an encoded frame.
3222 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3223 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003224 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003225
3226 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003227 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003228 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3229 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003230 EXPECT_THAT(new_video_source.sink_wants(),
3231 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003232
3233 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003234 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003235 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003236
mflodmancc3d4422017-08-03 08:27:51 -07003237 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003238}
3239
mflodmancc3d4422017-08-03 08:27:51 -07003240TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003241 const int kWidth = 1280;
3242 const int kHeight = 720;
3243 const size_t kNumFrames = 10;
3244
Henrik Boström381d1092020-05-12 18:49:07 +02003245 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003246 DataRate::BitsPerSec(kTargetBitrateBps),
3247 DataRate::BitsPerSec(kTargetBitrateBps),
3248 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003249
asaperssond0de2952017-04-21 01:47:31 -07003250 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003251 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003252 video_source_.set_adaptation_enabled(true);
3253
3254 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3255 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3256
3257 int downscales = 0;
3258 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003259 video_source_.IncomingCapturedFrame(
3260 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3261 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003262
asaperssonfab67072017-04-04 05:51:49 -07003263 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003264 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003265 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003266 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003267
3268 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3269 ++downscales;
3270
3271 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3272 EXPECT_EQ(downscales,
3273 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3274 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003275 }
mflodmancc3d4422017-08-03 08:27:51 -07003276 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003277}
3278
mflodmancc3d4422017-08-03 08:27:51 -07003279TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003280 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3281 const int kWidth = 1280;
3282 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003284 DataRate::BitsPerSec(kTargetBitrateBps),
3285 DataRate::BitsPerSec(kTargetBitrateBps),
3286 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003287
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003288 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003289 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003290 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003291 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003292 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003293
Åsa Persson8c1bf952018-09-13 10:42:19 +02003294 int64_t timestamp_ms = kFrameIntervalMs;
3295 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003296 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003297 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003298 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3299 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3300
3301 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003302 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003303 timestamp_ms += kFrameIntervalMs;
3304 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3305 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003306 EXPECT_THAT(source.sink_wants(),
3307 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003308 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3310
3311 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003312 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003313 timestamp_ms += kFrameIntervalMs;
3314 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003315 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003316 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003317 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3318 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3319
3320 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003321 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003322 timestamp_ms += kFrameIntervalMs;
3323 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3324 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003325 EXPECT_THAT(source.sink_wants(),
3326 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003327 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3328 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3329
3330 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003331 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003332 timestamp_ms += kFrameIntervalMs;
3333 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003334 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003335 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003336 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3337 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3338
mflodmancc3d4422017-08-03 08:27:51 -07003339 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003340}
3341
mflodmancc3d4422017-08-03 08:27:51 -07003342TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003343 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3344 const int kWidth = 1280;
3345 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003347 DataRate::BitsPerSec(kTargetBitrateBps),
3348 DataRate::BitsPerSec(kTargetBitrateBps),
3349 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003350
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003351 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003352 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003353 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003354 video_stream_encoder_->SetSource(&source,
3355 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003356
Åsa Persson8c1bf952018-09-13 10:42:19 +02003357 int64_t timestamp_ms = kFrameIntervalMs;
3358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003359 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003360 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003361 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3362 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
3364 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003365 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003366 timestamp_ms += kFrameIntervalMs;
3367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3368 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003369 EXPECT_THAT(source.sink_wants(),
3370 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3372 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3373
3374 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003375 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003376 timestamp_ms += kFrameIntervalMs;
3377 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003378 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003379 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003380 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3381 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003384 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003385 timestamp_ms += kFrameIntervalMs;
3386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3387 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003388 EXPECT_THAT(source.sink_wants(),
3389 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003390 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3391 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3392
3393 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003394 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003395 timestamp_ms += kFrameIntervalMs;
3396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003397 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003398 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003399 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3400 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3401
mflodmancc3d4422017-08-03 08:27:51 -07003402 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003403}
3404
Sergey Silkin41c650b2019-10-14 13:12:19 +02003405TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3406 fake_encoder_.SetResolutionBitrateLimits(
3407 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3408
Henrik Boström381d1092020-05-12 18:49:07 +02003409 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003410 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3411 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3412 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3413 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003414
3415 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003416 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003417 source.set_adaptation_enabled(true);
3418 video_stream_encoder_->SetSource(
3419 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3420
3421 // Insert 720p frame.
3422 int64_t timestamp_ms = kFrameIntervalMs;
3423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3424 WaitForEncodedFrame(1280, 720);
3425
3426 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003428 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3429 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3430 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3431 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003432 video_stream_encoder_->TriggerQualityLow();
3433
3434 // Insert 720p frame. It should be downscaled and encoded.
3435 timestamp_ms += kFrameIntervalMs;
3436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3437 WaitForEncodedFrame(960, 540);
3438
3439 // Trigger adapt up. Higher resolution should not be requested duo to lack
3440 // of bitrate.
3441 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003442 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003443
3444 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003446 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3447 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3448 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3449 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003450
3451 // Trigger adapt up. Higher resolution should be requested.
3452 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003453 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003454
3455 video_stream_encoder_->Stop();
3456}
3457
3458TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3459 fake_encoder_.SetResolutionBitrateLimits(
3460 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3461
3462 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003463 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003464 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3465 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3466 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3467 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003468
3469 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003470 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003471 source.set_adaptation_enabled(true);
3472 video_stream_encoder_->SetSource(
3473 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3474
3475 // Insert 720p frame. It should be dropped and lower resolution should be
3476 // requested.
3477 int64_t timestamp_ms = kFrameIntervalMs;
3478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3479 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003480 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003481
3482 // Insert 720p frame. It should be downscaled and encoded.
3483 timestamp_ms += kFrameIntervalMs;
3484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3485 WaitForEncodedFrame(960, 540);
3486
3487 video_stream_encoder_->Stop();
3488}
3489
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003490class BalancedDegradationTest : public VideoStreamEncoderTest {
3491 protected:
3492 void SetupTest() {
3493 // Reset encoder for field trials to take effect.
3494 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003495 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003496
3497 // Enable BALANCED preference.
3498 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003499 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3500 }
3501
3502 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003504 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3505 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003506 }
3507
Åsa Persson45b176f2019-09-30 11:19:05 +02003508 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003509 timestamp_ms_ += kFrameIntervalMs;
3510 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003511 }
3512
3513 void InsertFrameAndWaitForEncoded() {
3514 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003515 sink_.WaitForEncodedFrame(timestamp_ms_);
3516 }
3517
3518 const int kWidth = 640; // pixels:640x360=230400
3519 const int kHeight = 360;
3520 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3521 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003522 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003523};
3524
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003525TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003526 test::ScopedFieldTrials field_trials(
3527 "WebRTC-Video-BalancedDegradationSettings/"
3528 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3529 SetupTest();
3530
3531 // Force input frame rate.
3532 const int kInputFps = 24;
3533 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3534 stats.input_frame_rate = kInputFps;
3535 stats_proxy_->SetMockStats(stats);
3536
Åsa Persson45b176f2019-09-30 11:19:05 +02003537 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003538 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003539
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003540 // Trigger adapt down, expect scaled down framerate and resolution,
3541 // since Fps diff (input-requested:0) < threshold.
3542 video_stream_encoder_->TriggerQualityLow();
3543 EXPECT_THAT(source_.sink_wants(),
3544 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003545
3546 video_stream_encoder_->Stop();
3547}
3548
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003549TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003550 test::ScopedFieldTrials field_trials(
3551 "WebRTC-Video-BalancedDegradationSettings/"
3552 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3553 SetupTest();
3554
3555 // Force input frame rate.
3556 const int kInputFps = 25;
3557 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3558 stats.input_frame_rate = kInputFps;
3559 stats_proxy_->SetMockStats(stats);
3560
Åsa Persson45b176f2019-09-30 11:19:05 +02003561 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003562 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003563
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003564 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3565 // Fps diff (input-requested:1) == threshold.
3566 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003567 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003568
3569 video_stream_encoder_->Stop();
3570}
3571
3572TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3573 test::ScopedFieldTrials field_trials(
3574 "WebRTC-Video-BalancedDegradationSettings/"
3575 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3576 SetupTest();
3577
3578 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3579
Åsa Persson45b176f2019-09-30 11:19:05 +02003580 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003581 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003582
3583 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3584 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003585 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003586
3587 video_stream_encoder_->Stop();
3588}
3589
Åsa Perssonccfb3402019-09-25 15:13:04 +02003590TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003591 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003592 "WebRTC-Video-BalancedDegradationSettings/"
3593 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003594 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003595
Åsa Persson1b247f12019-08-14 17:26:39 +02003596 const int kMinBitrateBps = 425000;
3597 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003598 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003599
Åsa Persson45b176f2019-09-30 11:19:05 +02003600 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003601 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003602 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3603
3604 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3605 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003606 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003607 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003608 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3609
3610 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3611 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003612 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003613 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003614 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615
Åsa Persson30ab0152019-08-27 12:22:33 +02003616 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3617 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003618 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003619 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003620 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003621 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3622
3623 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003624 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003625 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003626 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003627
Åsa Persson30ab0152019-08-27 12:22:33 +02003628 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003629 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003630 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003631 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003632 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003633 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3634
3635 video_stream_encoder_->Stop();
3636}
3637
Åsa Perssonccfb3402019-09-25 15:13:04 +02003638TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003639 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3640 test::ScopedFieldTrials field_trials(
3641 "WebRTC-Video-BalancedDegradationSettings/"
3642 "pixels:57600|129600|230400,fps:7|24|24/");
3643 SetupTest();
3644 OnBitrateUpdated(kLowTargetBitrateBps);
3645
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003646 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003647
3648 // Insert frame, expect scaled down:
3649 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3650 InsertFrame();
3651 EXPECT_FALSE(WaitForFrame(1000));
3652 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3653 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3654
3655 // Insert frame, expect scaled down:
3656 // resolution (320x180@24fps).
3657 InsertFrame();
3658 EXPECT_FALSE(WaitForFrame(1000));
3659 EXPECT_LT(source_.sink_wants().max_pixel_count,
3660 source_.last_wants().max_pixel_count);
3661 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3662
3663 // Frame should not be dropped (min pixels per frame reached).
3664 InsertFrameAndWaitForEncoded();
3665
3666 video_stream_encoder_->Stop();
3667}
3668
3669TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003670 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003671 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003672 "WebRTC-Video-BalancedDegradationSettings/"
3673 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003674 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003675
Åsa Persson30ab0152019-08-27 12:22:33 +02003676 const int kResolutionMinBitrateBps = 435000;
3677 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003678 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003679
Åsa Persson45b176f2019-09-30 11:19:05 +02003680 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003681 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003682 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3683
3684 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3685 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003686 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003687 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003688 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3689
3690 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3691 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003692 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003693 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003694 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3695
3696 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3697 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003698 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003699 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003700 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3701
Åsa Persson30ab0152019-08-27 12:22:33 +02003702 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3703 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003704 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003705 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003706 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3707
3708 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3709 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003710 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003711 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3712
3713 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003714 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003715 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003716 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003717 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003718 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3719
3720 video_stream_encoder_->Stop();
3721}
3722
Åsa Perssonccfb3402019-09-25 15:13:04 +02003723TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003724 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003725 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003726 "WebRTC-Video-BalancedDegradationSettings/"
3727 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003728 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003729
Åsa Persson30ab0152019-08-27 12:22:33 +02003730 const int kMinBitrateBps = 425000;
3731 const int kTooLowMinBitrateBps = 424000;
3732 const int kResolutionMinBitrateBps = 435000;
3733 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003734 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003735
Åsa Persson45b176f2019-09-30 11:19:05 +02003736 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003737 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003738 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3739
3740 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3741 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003742 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003743 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003744 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3745
3746 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3747 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003748 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003749 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003750 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3751
3752 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3753 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003754 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003755 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003756 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3757
3758 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3759 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003760 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003761 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3762
3763 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003764 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003765 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003766 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003767 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003768 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3769
3770 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003771 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003772 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003773 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003774 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3775
3776 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003777 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003778 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003779 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003780 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003781 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3782
Åsa Persson1b247f12019-08-14 17:26:39 +02003783 video_stream_encoder_->Stop();
3784}
3785
mflodmancc3d4422017-08-03 08:27:51 -07003786TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003787 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3788 const int kWidth = 1280;
3789 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003791 DataRate::BitsPerSec(kTargetBitrateBps),
3792 DataRate::BitsPerSec(kTargetBitrateBps),
3793 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003794
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003795 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003796 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003797 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003798 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003799 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003800
Åsa Persson8c1bf952018-09-13 10:42:19 +02003801 int64_t timestamp_ms = kFrameIntervalMs;
3802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003803 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003805 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3807 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3808 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3809
3810 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003811 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003812 timestamp_ms += kFrameIntervalMs;
3813 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3814 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003815 EXPECT_THAT(source.sink_wants(),
3816 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003817 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3818 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3819 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3820 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3821
3822 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003823 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003824 timestamp_ms += kFrameIntervalMs;
3825 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3826 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003827 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003828 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3830 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3831 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3832
Jonathan Yubc771b72017-12-08 17:04:29 -08003833 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003834 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003835 timestamp_ms += kFrameIntervalMs;
3836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3837 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003838 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003839 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003841 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003842 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3843
Jonathan Yubc771b72017-12-08 17:04:29 -08003844 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003845 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003846 timestamp_ms += kFrameIntervalMs;
3847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3848 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003849 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003850 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003851 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3852 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3853 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3854 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3855
Jonathan Yubc771b72017-12-08 17:04:29 -08003856 // Trigger quality adapt down, expect no change (min resolution reached).
3857 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003858 timestamp_ms += kFrameIntervalMs;
3859 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3860 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003861 EXPECT_THAT(source.sink_wants(), FpsMax());
3862 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003863 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3864 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3865 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3866 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3867
Evan Shrubsole64469032020-06-11 10:45:29 +02003868 // Trigger quality adapt up, expect upscaled resolution (480x270).
3869 video_stream_encoder_->TriggerQualityHigh();
3870 timestamp_ms += kFrameIntervalMs;
3871 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3872 WaitForEncodedFrame(timestamp_ms);
3873 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3874 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3875 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3876 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3877 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3878
3879 // Trigger quality and cpu adapt up since both are most limited, expect
3880 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003881 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003882 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003883 timestamp_ms += kFrameIntervalMs;
3884 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3885 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003886 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003887 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3888 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3889 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003890 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003891
Evan Shrubsole64469032020-06-11 10:45:29 +02003892 // Trigger quality and cpu adapt up since both are most limited, expect
3893 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003894 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003895 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 timestamp_ms += kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3898 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003899 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003900 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003901 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003902 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003903 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3904 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003905
Evan Shrubsole64469032020-06-11 10:45:29 +02003906 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3907 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003908 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003909 timestamp_ms += kFrameIntervalMs;
3910 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3911 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003912 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003913 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3914 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003915 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003916 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003917
3918 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003919 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003920 timestamp_ms += kFrameIntervalMs;
3921 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003922 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003923 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003924 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003925 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3926 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003927 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003928 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003929
mflodmancc3d4422017-08-03 08:27:51 -07003930 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003931}
3932
mflodmancc3d4422017-08-03 08:27:51 -07003933TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003934 const int kWidth = 640;
3935 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003936
Henrik Boström381d1092020-05-12 18:49:07 +02003937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003938 DataRate::BitsPerSec(kTargetBitrateBps),
3939 DataRate::BitsPerSec(kTargetBitrateBps),
3940 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003941
perkj803d97f2016-11-01 11:45:46 -07003942 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003943 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003944 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003945 }
3946
mflodmancc3d4422017-08-03 08:27:51 -07003947 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003948 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003949 video_source_.IncomingCapturedFrame(CreateFrame(
3950 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003951 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003952 }
3953
mflodmancc3d4422017-08-03 08:27:51 -07003954 video_stream_encoder_->Stop();
3955 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003956 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003957
Ying Wangef3998f2019-12-09 13:06:53 +01003958 EXPECT_METRIC_EQ(
3959 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3960 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003961 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3962}
3963
mflodmancc3d4422017-08-03 08:27:51 -07003964TEST_F(VideoStreamEncoderTest,
3965 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003967 DataRate::BitsPerSec(kTargetBitrateBps),
3968 DataRate::BitsPerSec(kTargetBitrateBps),
3969 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003970 const int kWidth = 640;
3971 const int kHeight = 360;
3972
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003973 video_stream_encoder_->SetSource(&video_source_,
3974 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003975
3976 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3977 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003978 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003979 }
3980
mflodmancc3d4422017-08-03 08:27:51 -07003981 video_stream_encoder_->Stop();
3982 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003983 stats_proxy_.reset();
3984
3985 EXPECT_EQ(0,
3986 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3987}
3988
Per Kjellanderdcef6412020-10-07 15:09:05 +02003989TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
3990 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
3991 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
3992 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08003993
3994 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003995 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003996 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003997 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3998 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003999
Henrik Boström381d1092020-05-12 18:49:07 +02004000 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004001 DataRate::BitsPerSec(kLowTargetBitrateBps),
4002 DataRate::BitsPerSec(kLowTargetBitrateBps),
4003 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004004
sprang57c2fff2017-01-16 06:24:02 -08004005 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004006 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4007 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004008 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4009 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4010
Erik Språngd7329ca2019-02-21 21:19:53 +01004011 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004012 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004013 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004014
Per Kjellanderdcef6412020-10-07 15:09:05 +02004015 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004016 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004017 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4018 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004019 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004020 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004021
Per Kjellanderdcef6412020-10-07 15:09:05 +02004022 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004023 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004024 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004025 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004026 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4027 WaitForEncodedFrame(CurrentTimeMs());
4028 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004029 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004030 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004031
mflodmancc3d4422017-08-03 08:27:51 -07004032 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004033}
4034
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004035TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004036 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
4037 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4038 kVideoLayersAllocation);
4039
4040 const int kDefaultFps = 30;
4041
4042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4043 DataRate::BitsPerSec(kLowTargetBitrateBps),
4044 DataRate::BitsPerSec(kLowTargetBitrateBps),
4045 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4046
4047 video_source_.IncomingCapturedFrame(
4048 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4049 WaitForEncodedFrame(CurrentTimeMs());
4050 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4051 VideoLayersAllocation last_layer_allocation =
4052 sink_.GetLastVideoLayersAllocation();
4053 // kLowTargetBitrateBps is only enough for one spatial layer.
4054 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4055
4056 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004057 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004058 // Check that encoder has been updated too, not just allocation observer.
4059 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4060 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4061
Erik Språng9d69cbe2020-10-22 17:44:42 +02004062 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004063 int number_of_layers_allocation = 1;
4064 const int64_t start_time_ms = CurrentTimeMs();
4065 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4066 video_source_.IncomingCapturedFrame(
4067 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4068 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004069 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4070 number_of_layers_allocation = sink_.number_of_layers_allocations();
4071 VideoLayersAllocation new_allocation =
4072 sink_.GetLastVideoLayersAllocation();
4073 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4074 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4075 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4076 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4077 .target_bitrate_per_temporal_layer,
4078 last_layer_allocation.active_spatial_layers[0]
4079 .target_bitrate_per_temporal_layer);
4080 last_layer_allocation = new_allocation;
4081 }
4082 }
4083 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4084 video_stream_encoder_->Stop();
4085}
4086
4087TEST_F(VideoStreamEncoderTest,
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004088 ReportsVideoLayersAllocationForVP8WithMidleLayerDisabled) {
4089 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4090 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4091 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
4092 video_send_config_.encoder_settings.allocation_cb_type =
4093 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4094 kVideoLayersAllocation;
4095 VideoEncoderConfig video_encoder_config;
4096 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4097 /* num_streams*/ 3, &video_encoder_config);
4098 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4099 video_encoder_config.content_type =
4100 VideoEncoderConfig::ContentType::kRealtimeVideo;
4101 video_encoder_config.encoder_specific_settings =
4102 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4103 VideoEncoder::GetDefaultVp8Settings());
4104 for (auto& layer : video_encoder_config.simulcast_layers) {
4105 layer.num_temporal_layers = 2;
4106 }
4107 // Simulcast layers are used for enabling/disabling streams.
4108 video_encoder_config.simulcast_layers[0].active = true;
4109 video_encoder_config.simulcast_layers[1].active = false;
4110 video_encoder_config.simulcast_layers[2].active = true;
4111 ConfigureEncoder(std::move(video_encoder_config));
4112
4113 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4114 DataRate::BitsPerSec(kTargetBitrateBps),
4115 DataRate::BitsPerSec(kTargetBitrateBps),
4116 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4117
4118 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4119 WaitForEncodedFrame(CurrentTimeMs());
4120 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4121 VideoLayersAllocation last_layer_allocation =
4122 sink_.GetLastVideoLayersAllocation();
4123
4124 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4125 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4126 .target_bitrate_per_temporal_layer,
4127 SizeIs(2));
4128 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4129 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4130 video_stream_encoder_->Stop();
4131}
4132
4133TEST_F(VideoStreamEncoderTest,
4134 ReportsVideoLayersAllocationForVP8WithMidleAndHighestLayerDisabled) {
4135 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4136 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4137 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
4138 video_send_config_.encoder_settings.allocation_cb_type =
4139 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4140 kVideoLayersAllocation;
4141 VideoEncoderConfig video_encoder_config;
4142 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4143 /* num_streams*/ 3, &video_encoder_config);
4144 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4145 video_encoder_config.content_type =
4146 VideoEncoderConfig::ContentType::kRealtimeVideo;
4147 video_encoder_config.encoder_specific_settings =
4148 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4149 VideoEncoder::GetDefaultVp8Settings());
4150 for (auto& layer : video_encoder_config.simulcast_layers) {
4151 layer.num_temporal_layers = 2;
4152 }
4153 // Simulcast layers are used for enabling/disabling streams.
4154 video_encoder_config.simulcast_layers[0].active = true;
4155 video_encoder_config.simulcast_layers[1].active = false;
4156 video_encoder_config.simulcast_layers[2].active = false;
4157 ConfigureEncoder(std::move(video_encoder_config));
4158
4159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4160 DataRate::BitsPerSec(kTargetBitrateBps),
4161 DataRate::BitsPerSec(kTargetBitrateBps),
4162 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4163
4164 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4165 WaitForEncodedFrame(CurrentTimeMs());
4166 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4167 VideoLayersAllocation last_layer_allocation =
4168 sink_.GetLastVideoLayersAllocation();
4169
4170 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4171 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4172 .target_bitrate_per_temporal_layer,
4173 SizeIs(2));
4174 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4175
4176 video_stream_encoder_->Stop();
4177}
4178
4179TEST_F(VideoStreamEncoderTest,
4180 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4181 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4182 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4183 video_send_config_.encoder_settings.allocation_cb_type =
4184 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4185 kVideoLayersAllocation;
4186 VideoEncoderConfig video_encoder_config;
4187 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4188 /* num_streams*/ 1, &video_encoder_config);
4189 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4190 video_encoder_config.content_type =
4191 VideoEncoderConfig::ContentType::kRealtimeVideo;
4192 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4193 vp9_settings.numberOfSpatialLayers = 2;
4194 vp9_settings.numberOfTemporalLayers = 2;
4195 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4196 vp9_settings.automaticResizeOn = false;
4197 video_encoder_config.encoder_specific_settings =
4198 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4199 vp9_settings);
4200 ConfigureEncoder(std::move(video_encoder_config));
4201
4202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4203 DataRate::BitsPerSec(kTargetBitrateBps),
4204 DataRate::BitsPerSec(kTargetBitrateBps),
4205 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4206
4207 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4208 WaitForEncodedFrame(CurrentTimeMs());
4209 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4210 VideoLayersAllocation last_layer_allocation =
4211 sink_.GetLastVideoLayersAllocation();
4212
4213 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4214 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4215 .target_bitrate_per_temporal_layer,
4216 SizeIs(2));
4217 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4218 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4219 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4220 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4221 .target_bitrate_per_temporal_layer,
4222 SizeIs(2));
4223 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4224 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4225 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4226
4227 // Since full SVC is used, expect the top layer to utilize the full target
4228 // rate.
4229 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4230 .target_bitrate_per_temporal_layer[1],
4231 DataRate::BitsPerSec(kTargetBitrateBps));
4232 video_stream_encoder_->Stop();
4233}
4234
4235TEST_F(VideoStreamEncoderTest,
4236 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4237 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4238 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
4239 video_send_config_.encoder_settings.allocation_cb_type =
4240 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4241 kVideoLayersAllocation;
4242 VideoEncoderConfig video_encoder_config;
4243 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4244 /* num_streams*/ 1, &video_encoder_config);
4245 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4246 video_encoder_config.content_type =
4247 VideoEncoderConfig::ContentType::kRealtimeVideo;
4248 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4249 vp9_settings.numberOfSpatialLayers = 2;
4250 vp9_settings.numberOfTemporalLayers = 2;
4251 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4252 vp9_settings.automaticResizeOn = false;
4253 video_encoder_config.encoder_specific_settings =
4254 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4255 vp9_settings);
4256 ConfigureEncoder(std::move(video_encoder_config));
4257
4258 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4259 DataRate::BitsPerSec(kTargetBitrateBps),
4260 DataRate::BitsPerSec(kTargetBitrateBps),
4261 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4262
4263 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4264 WaitForEncodedFrame(CurrentTimeMs());
4265 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4266 VideoLayersAllocation last_layer_allocation =
4267 sink_.GetLastVideoLayersAllocation();
4268
4269 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4270 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4271 .target_bitrate_per_temporal_layer,
4272 SizeIs(1));
4273 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4274 .target_bitrate_per_temporal_layer,
4275 SizeIs(1));
4276 // Since full SVC is used, expect the top layer to utilize the full target
4277 // rate.
4278 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4279 .target_bitrate_per_temporal_layer[0],
4280 DataRate::BitsPerSec(kTargetBitrateBps));
4281 video_stream_encoder_->Stop();
4282}
4283
4284TEST_F(VideoStreamEncoderTest,
4285 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4286 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4287 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4288 video_send_config_.encoder_settings.allocation_cb_type =
4289 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4290 kVideoLayersAllocation;
4291 VideoEncoderConfig video_encoder_config;
4292 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4293 /* num_streams*/ 1, &video_encoder_config);
4294 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4295 video_encoder_config.content_type =
4296 VideoEncoderConfig::ContentType::kRealtimeVideo;
4297 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4298 vp9_settings.numberOfSpatialLayers = 2;
4299 vp9_settings.numberOfTemporalLayers = 2;
4300 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4301 vp9_settings.automaticResizeOn = false;
4302 video_encoder_config.encoder_specific_settings =
4303 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4304 vp9_settings);
4305 ConfigureEncoder(std::move(video_encoder_config));
4306
4307 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4308 DataRate::BitsPerSec(kTargetBitrateBps),
4309 DataRate::BitsPerSec(kTargetBitrateBps),
4310 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4311
4312 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4313 WaitForEncodedFrame(CurrentTimeMs());
4314 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4315 VideoLayersAllocation last_layer_allocation =
4316 sink_.GetLastVideoLayersAllocation();
4317
4318 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4319 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4320 .target_bitrate_per_temporal_layer,
4321 SizeIs(2));
4322 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4323 .target_bitrate_per_temporal_layer,
4324 SizeIs(2));
4325 // Since KSVC is, spatial layers are independend except on key frames.
4326 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4327 .target_bitrate_per_temporal_layer[1],
4328 DataRate::BitsPerSec(kTargetBitrateBps));
4329 video_stream_encoder_->Stop();
4330}
4331
4332TEST_F(VideoStreamEncoderTest,
4333 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4334 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4335 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4336 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
4337 video_send_config_.encoder_settings.allocation_cb_type =
4338 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4339 kVideoLayersAllocation;
4340 VideoEncoderConfig video_encoder_config;
4341 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4342 /* num_streams*/ 1, &video_encoder_config);
4343 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4344 video_encoder_config.content_type =
4345 VideoEncoderConfig::ContentType::kRealtimeVideo;
4346 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4347 vp9_settings.numberOfSpatialLayers = 3;
4348 vp9_settings.numberOfTemporalLayers = 2;
4349 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4350 vp9_settings.automaticResizeOn = false;
4351 video_encoder_config.encoder_specific_settings =
4352 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4353 vp9_settings);
4354 // Simulcast layers are used for enabling/disabling streams.
4355 video_encoder_config.simulcast_layers.resize(3);
4356 video_encoder_config.simulcast_layers[0].active = false;
4357 video_encoder_config.simulcast_layers[1].active = true;
4358 video_encoder_config.simulcast_layers[2].active = true;
4359 ConfigureEncoder(std::move(video_encoder_config));
4360
4361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4362 DataRate::BitsPerSec(kTargetBitrateBps),
4363 DataRate::BitsPerSec(kTargetBitrateBps),
4364 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4365
4366 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4367 WaitForEncodedFrame(CurrentTimeMs());
4368 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4369 VideoLayersAllocation last_layer_allocation =
4370 sink_.GetLastVideoLayersAllocation();
4371
4372 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4373 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4374 .target_bitrate_per_temporal_layer,
4375 SizeIs(2));
4376 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4377 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4378
4379 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4380 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4381 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4382 .target_bitrate_per_temporal_layer,
4383 SizeIs(2));
4384 // Since full SVC is used, expect the top layer to utilize the full target
4385 // rate.
4386 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4387 .target_bitrate_per_temporal_layer[1],
4388 DataRate::BitsPerSec(kTargetBitrateBps));
4389 video_stream_encoder_->Stop();
4390}
4391
4392TEST_F(VideoStreamEncoderTest,
4393 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4394 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4395 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4396 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
4397 video_send_config_.encoder_settings.allocation_cb_type =
4398 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4399 kVideoLayersAllocation;
4400 VideoEncoderConfig video_encoder_config;
4401 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4402 /* num_streams*/ 1, &video_encoder_config);
4403 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4404 video_encoder_config.content_type =
4405 VideoEncoderConfig::ContentType::kRealtimeVideo;
4406 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4407 vp9_settings.numberOfSpatialLayers = 3;
4408 vp9_settings.numberOfTemporalLayers = 2;
4409 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4410 vp9_settings.automaticResizeOn = false;
4411 video_encoder_config.encoder_specific_settings =
4412 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4413 vp9_settings);
4414 // Simulcast layers are used for enabling/disabling streams.
4415 video_encoder_config.simulcast_layers.resize(3);
4416 video_encoder_config.simulcast_layers[2].active = false;
4417 ConfigureEncoder(std::move(video_encoder_config));
4418
4419 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4420 DataRate::BitsPerSec(kTargetBitrateBps),
4421 DataRate::BitsPerSec(kTargetBitrateBps),
4422 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4423
4424 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4425 WaitForEncodedFrame(CurrentTimeMs());
4426 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4427 VideoLayersAllocation last_layer_allocation =
4428 sink_.GetLastVideoLayersAllocation();
4429
4430 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4431 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4432 .target_bitrate_per_temporal_layer,
4433 SizeIs(2));
4434 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4435 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4436
4437 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4438 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4439 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4440 .target_bitrate_per_temporal_layer,
4441 SizeIs(2));
4442 video_stream_encoder_->Stop();
4443}
4444
4445TEST_F(VideoStreamEncoderTest,
4446 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4447 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4448 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4449 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
4450 video_send_config_.encoder_settings.allocation_cb_type =
4451 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4452 kVideoLayersAllocation;
4453 VideoEncoderConfig video_encoder_config;
4454 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4455 /* num_streams*/ 1, &video_encoder_config);
4456 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4457 video_encoder_config.content_type =
4458 VideoEncoderConfig::ContentType::kRealtimeVideo;
4459 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4460 vp9_settings.numberOfSpatialLayers = 3;
4461 vp9_settings.numberOfTemporalLayers = 2;
4462 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4463 vp9_settings.automaticResizeOn = false;
4464 video_encoder_config.encoder_specific_settings =
4465 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4466 vp9_settings);
4467 // Simulcast layers are used for enabling/disabling streams.
4468 video_encoder_config.simulcast_layers.resize(3);
4469 video_encoder_config.simulcast_layers[0].active = false;
4470 video_encoder_config.simulcast_layers[1].active = false;
4471 video_encoder_config.simulcast_layers[2].active = true;
4472 ConfigureEncoder(std::move(video_encoder_config));
4473
4474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4475 DataRate::BitsPerSec(kTargetBitrateBps),
4476 DataRate::BitsPerSec(kTargetBitrateBps),
4477 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4478
4479 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4480 WaitForEncodedFrame(CurrentTimeMs());
4481 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4482 VideoLayersAllocation last_layer_allocation =
4483 sink_.GetLastVideoLayersAllocation();
4484
4485 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4486 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4487 .target_bitrate_per_temporal_layer,
4488 SizeIs(2));
4489 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4490 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4491 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4492 .target_bitrate_per_temporal_layer[1],
4493 DataRate::BitsPerSec(kTargetBitrateBps));
4494 video_stream_encoder_->Stop();
4495}
4496
4497TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4498 ResetEncoder("H264", 1, 1, 1, false,
4499 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4500 kVideoLayersAllocation);
4501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4502 DataRate::BitsPerSec(kTargetBitrateBps),
4503 DataRate::BitsPerSec(kTargetBitrateBps),
4504 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4505
4506 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4507 WaitForEncodedFrame(CurrentTimeMs());
4508 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4509 VideoLayersAllocation last_layer_allocation =
4510 sink_.GetLastVideoLayersAllocation();
4511
4512 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4513 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4514 .target_bitrate_per_temporal_layer,
4515 SizeIs(1));
4516 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4517 .target_bitrate_per_temporal_layer[0],
4518 DataRate::BitsPerSec(kTargetBitrateBps));
4519 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4520 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4521 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4522 video_stream_encoder_->Stop();
4523}
4524
4525TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004526 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4527 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
4528 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4529 kVideoLayersAllocation);
4530
4531 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4532 DataRate::BitsPerSec(kLowTargetBitrateBps),
4533 DataRate::BitsPerSec(kLowTargetBitrateBps),
4534 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4535
4536 video_source_.IncomingCapturedFrame(
4537 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4538 WaitForEncodedFrame(CurrentTimeMs());
4539 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4540 VideoLayersAllocation last_layer_allocation =
4541 sink_.GetLastVideoLayersAllocation();
4542 // kLowTargetBitrateBps is only enough for one spatial layer.
4543 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4544 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4545 .target_bitrate_per_temporal_layer[0],
4546 DataRate::BitsPerSec(kLowTargetBitrateBps));
4547
4548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4549 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4550 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4551 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4552 video_source_.IncomingCapturedFrame(
4553 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4554 WaitForEncodedFrame(CurrentTimeMs());
4555
4556 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4557 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4558 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4559 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4560 .target_bitrate_per_temporal_layer[0],
4561 DataRate::Zero());
4562
4563 video_stream_encoder_->Stop();
4564}
4565
Per Kjellander4190ce92020-12-15 17:24:55 +01004566TEST_F(VideoStreamEncoderTest,
4567 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4568 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
4569 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4570 kVideoLayersAllocation);
4571
4572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4573 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4574 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4575 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4576
4577 video_source_.IncomingCapturedFrame(
4578 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4579 WaitForEncodedFrame(CurrentTimeMs());
4580 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4581 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4582 SizeIs(2));
4583 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4584 codec_width_);
4585 EXPECT_EQ(
4586 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4587 codec_height_);
4588
4589 video_source_.IncomingCapturedFrame(
4590 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4591 WaitForEncodedFrame(CurrentTimeMs());
4592 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4593 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4594 SizeIs(2));
4595 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4596 codec_width_ / 2);
4597 EXPECT_EQ(
4598 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4599 codec_height_ / 2);
4600
4601 video_stream_encoder_->Stop();
4602}
4603
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004604TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4605 // 2 TLs configured, temporal layers supported by encoder.
4606 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004607 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
4608 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4609 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004610 fake_encoder_.SetTemporalLayersSupported(0, true);
4611
4612 // Bitrate allocated across temporal layers.
4613 const int kTl0Bps = kTargetBitrateBps *
4614 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004615 kNumTemporalLayers, /*temporal_id*/ 0,
4616 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004617 const int kTl1Bps = kTargetBitrateBps *
4618 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004619 kNumTemporalLayers, /*temporal_id*/ 1,
4620 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004621 VideoBitrateAllocation expected_bitrate;
4622 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4623 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4624
4625 VerifyAllocatedBitrate(expected_bitrate);
4626 video_stream_encoder_->Stop();
4627}
4628
4629TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4630 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004631 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
4632 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4633 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004634 fake_encoder_.SetTemporalLayersSupported(0, false);
4635
4636 // Temporal layers not supported by the encoder.
4637 // Total bitrate should be at ti:0.
4638 VideoBitrateAllocation expected_bitrate;
4639 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4640
4641 VerifyAllocatedBitrate(expected_bitrate);
4642 video_stream_encoder_->Stop();
4643}
4644
4645TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004646 webrtc::test::ScopedFieldTrials field_trials(
4647 "WebRTC-Video-QualityScalerSettings/"
4648 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4649 // Reset encoder for field trials to take effect.
4650 ConfigureEncoder(video_encoder_config_.Copy());
4651
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004652 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004653 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
4654 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
4655 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004656 fake_encoder_.SetTemporalLayersSupported(0, true);
4657 fake_encoder_.SetTemporalLayersSupported(1, false);
4658
4659 const int kS0Bps = 150000;
4660 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004661 kS0Bps *
4662 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4663 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004664 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004665 kS0Bps *
4666 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4667 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004668 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4669 // Temporal layers not supported by si:1.
4670 VideoBitrateAllocation expected_bitrate;
4671 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4672 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4673 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4674
4675 VerifyAllocatedBitrate(expected_bitrate);
4676 video_stream_encoder_->Stop();
4677}
4678
Niels Möller7dc26b72017-12-06 10:27:48 +01004679TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4680 const int kFrameWidth = 1280;
4681 const int kFrameHeight = 720;
4682 const int kFramerate = 24;
4683
Henrik Boström381d1092020-05-12 18:49:07 +02004684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004685 DataRate::BitsPerSec(kTargetBitrateBps),
4686 DataRate::BitsPerSec(kTargetBitrateBps),
4687 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004688 test::FrameForwarder source;
4689 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004690 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004691
4692 // Insert a single frame, triggering initial configuration.
4693 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4694 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4695
4696 EXPECT_EQ(
4697 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4698 kDefaultFramerate);
4699
4700 // Trigger reconfigure encoder (without resetting the entire instance).
4701 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004702 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4703 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004704 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004705 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004706 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004707 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4708
4709 // Detector should be updated with fps limit from codec config.
4710 EXPECT_EQ(
4711 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4712 kFramerate);
4713
4714 // Trigger overuse, max framerate should be reduced.
4715 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4716 stats.input_frame_rate = kFramerate;
4717 stats_proxy_->SetMockStats(stats);
4718 video_stream_encoder_->TriggerCpuOveruse();
4719 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4720 int adapted_framerate =
4721 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4722 EXPECT_LT(adapted_framerate, kFramerate);
4723
4724 // Trigger underuse, max framerate should go back to codec configured fps.
4725 // Set extra low fps, to make sure it's actually reset, not just incremented.
4726 stats = stats_proxy_->GetStats();
4727 stats.input_frame_rate = adapted_framerate / 2;
4728 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004729 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004730 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4731 EXPECT_EQ(
4732 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4733 kFramerate);
4734
4735 video_stream_encoder_->Stop();
4736}
4737
4738TEST_F(VideoStreamEncoderTest,
4739 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
4740 const int kFrameWidth = 1280;
4741 const int kFrameHeight = 720;
4742 const int kLowFramerate = 15;
4743 const int kHighFramerate = 25;
4744
Henrik Boström381d1092020-05-12 18:49:07 +02004745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004746 DataRate::BitsPerSec(kTargetBitrateBps),
4747 DataRate::BitsPerSec(kTargetBitrateBps),
4748 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004749 test::FrameForwarder source;
4750 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004751 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004752
4753 // Trigger initial configuration.
4754 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004755 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4756 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004757 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004758 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02004759 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02004760 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004761 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4762
4763 EXPECT_EQ(
4764 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4765 kLowFramerate);
4766
4767 // Trigger overuse, max framerate should be reduced.
4768 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4769 stats.input_frame_rate = kLowFramerate;
4770 stats_proxy_->SetMockStats(stats);
4771 video_stream_encoder_->TriggerCpuOveruse();
4772 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4773 int adapted_framerate =
4774 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4775 EXPECT_LT(adapted_framerate, kLowFramerate);
4776
4777 // Reconfigure the encoder with a new (higher max framerate), max fps should
4778 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02004779 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004780 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4781 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004782 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004783 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4784
4785 EXPECT_EQ(
4786 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4787 adapted_framerate);
4788
4789 // Trigger underuse, max framerate should go back to codec configured fps.
4790 stats = stats_proxy_->GetStats();
4791 stats.input_frame_rate = adapted_framerate;
4792 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004793 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004794 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4795 EXPECT_EQ(
4796 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4797 kHighFramerate);
4798
4799 video_stream_encoder_->Stop();
4800}
4801
mflodmancc3d4422017-08-03 08:27:51 -07004802TEST_F(VideoStreamEncoderTest,
4803 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07004804 const int kFrameWidth = 1280;
4805 const int kFrameHeight = 720;
4806 const int kFramerate = 24;
4807
Henrik Boström381d1092020-05-12 18:49:07 +02004808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004809 DataRate::BitsPerSec(kTargetBitrateBps),
4810 DataRate::BitsPerSec(kTargetBitrateBps),
4811 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07004812 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004813 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004814 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07004815
4816 // Trigger initial configuration.
4817 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004818 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4819 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07004820 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07004821 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004822 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004823 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004824 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004825
Niels Möller7dc26b72017-12-06 10:27:48 +01004826 EXPECT_EQ(
4827 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4828 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004829
4830 // Trigger overuse, max framerate should be reduced.
4831 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4832 stats.input_frame_rate = kFramerate;
4833 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004834 video_stream_encoder_->TriggerCpuOveruse();
4835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004836 int adapted_framerate =
4837 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004838 EXPECT_LT(adapted_framerate, kFramerate);
4839
4840 // Change degradation preference to not enable framerate scaling. Target
4841 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004842 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004843 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004844 EXPECT_EQ(
4845 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4846 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004847
mflodmancc3d4422017-08-03 08:27:51 -07004848 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004849}
4850
mflodmancc3d4422017-08-03 08:27:51 -07004851TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004852 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004854 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4855 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4856 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004857 const int kWidth = 640;
4858 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004859
asaperssonfab67072017-04-04 05:51:49 -07004860 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004861
4862 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004863 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004864
4865 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004866 EXPECT_TRUE_WAIT(
4867 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004868
sprangc5d62e22017-04-02 23:53:04 -07004869 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004870
asaperssonfab67072017-04-04 05:51:49 -07004871 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004872 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004873 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004874
4875 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004876 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004877
Henrik Boström2671dac2020-05-19 16:29:09 +02004878 EXPECT_TRUE_WAIT(
4879 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004880
mflodmancc3d4422017-08-03 08:27:51 -07004881 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004882}
4883
mflodmancc3d4422017-08-03 08:27:51 -07004884TEST_F(VideoStreamEncoderTest,
4885 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004886 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004888 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4889 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4890 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004891 const int kWidth = 640;
4892 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004893
4894 // We expect the n initial frames to get dropped.
4895 int i;
4896 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004897 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004898 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004899 }
4900 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004901 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004902 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004903
4904 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004905 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004906
mflodmancc3d4422017-08-03 08:27:51 -07004907 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004908}
4909
mflodmancc3d4422017-08-03 08:27:51 -07004910TEST_F(VideoStreamEncoderTest,
4911 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004912 const int kWidth = 640;
4913 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004914 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004915 DataRate::BitsPerSec(kLowTargetBitrateBps),
4916 DataRate::BitsPerSec(kLowTargetBitrateBps),
4917 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004918
4919 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004920 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004921 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004922
asaperssonfab67072017-04-04 05:51:49 -07004923 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004924 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004925 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004926
mflodmancc3d4422017-08-03 08:27:51 -07004927 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004928}
4929
mflodmancc3d4422017-08-03 08:27:51 -07004930TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004931 const int kWidth = 640;
4932 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004933 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004934
4935 VideoEncoderConfig video_encoder_config;
4936 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4937 // Make format different, to force recreation of encoder.
4938 video_encoder_config.video_format.parameters["foo"] = "foo";
4939 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004940 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004942 DataRate::BitsPerSec(kLowTargetBitrateBps),
4943 DataRate::BitsPerSec(kLowTargetBitrateBps),
4944 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004945
kthelgasonb83797b2017-02-14 11:57:25 -08004946 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004947 video_stream_encoder_->SetSource(&video_source_,
4948 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004949
asaperssonfab67072017-04-04 05:51:49 -07004950 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004951 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004952 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004953
mflodmancc3d4422017-08-03 08:27:51 -07004954 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004955 fake_encoder_.SetQualityScaling(true);
4956}
4957
Åsa Persson139f4dc2019-08-02 09:29:58 +02004958TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4959 webrtc::test::ScopedFieldTrials field_trials(
4960 "WebRTC-Video-QualityScalerSettings/"
4961 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4962 // Reset encoder for field trials to take effect.
4963 ConfigureEncoder(video_encoder_config_.Copy());
4964 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4965 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4966 const int kWidth = 640;
4967 const int kHeight = 360;
4968
Henrik Boström381d1092020-05-12 18:49:07 +02004969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004970 DataRate::BitsPerSec(kTargetBitrateBps),
4971 DataRate::BitsPerSec(kTargetBitrateBps),
4972 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004973 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4974 // Frame should not be dropped.
4975 WaitForEncodedFrame(1);
4976
Henrik Boström381d1092020-05-12 18:49:07 +02004977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004978 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4979 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4980 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004981 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4982 // Frame should not be dropped.
4983 WaitForEncodedFrame(2);
4984
Henrik Boström381d1092020-05-12 18:49:07 +02004985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004986 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4987 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4988 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004989 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4990 // Expect to drop this frame, the wait should time out.
4991 ExpectDroppedFrame();
4992
4993 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004994 EXPECT_TRUE_WAIT(
4995 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004996 video_stream_encoder_->Stop();
4997}
4998
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004999TEST_F(VideoStreamEncoderTest,
5000 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5001 webrtc::test::ScopedFieldTrials field_trials(
5002 "WebRTC-Video-QualityScalerSettings/"
5003 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5004 fake_encoder_.SetQualityScaling(false);
5005 ConfigureEncoder(video_encoder_config_.Copy());
5006 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5007 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5008 const int kWidth = 640;
5009 const int kHeight = 360;
5010
5011 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5012 DataRate::BitsPerSec(kTargetBitrateBps),
5013 DataRate::BitsPerSec(kTargetBitrateBps),
5014 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5015 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5016 // Frame should not be dropped.
5017 WaitForEncodedFrame(1);
5018
5019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5020 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5021 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5022 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5023 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5024 // Frame should not be dropped.
5025 WaitForEncodedFrame(2);
5026
5027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5028 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5029 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5030 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5031 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5032 // Not dropped since quality scaling is disabled.
5033 WaitForEncodedFrame(3);
5034
5035 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005036 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005037 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5038
5039 video_stream_encoder_->Stop();
5040}
5041
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005042TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5043 const int kLowTargetBitrateBps = 400000;
5044 // Set simulcast.
5045 ResetEncoder("VP8", 3, 1, 1, false);
5046 fake_encoder_.SetQualityScaling(true);
5047 const int kWidth = 1280;
5048 const int kHeight = 720;
5049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5050 DataRate::BitsPerSec(kLowTargetBitrateBps),
5051 DataRate::BitsPerSec(kLowTargetBitrateBps),
5052 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5053 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5054 // Frame should not be dropped.
5055 WaitForEncodedFrame(1);
5056
5057 // Trigger QVGA "singlecast"
5058 // Update the config.
5059 VideoEncoderConfig video_encoder_config;
5060 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5061 &video_encoder_config);
5062 for (auto& layer : video_encoder_config.simulcast_layers) {
5063 layer.num_temporal_layers = 1;
5064 layer.max_framerate = kDefaultFramerate;
5065 }
5066 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5067 video_encoder_config.content_type =
5068 VideoEncoderConfig::ContentType::kRealtimeVideo;
5069
5070 video_encoder_config.simulcast_layers[0].active = true;
5071 video_encoder_config.simulcast_layers[1].active = false;
5072 video_encoder_config.simulcast_layers[2].active = false;
5073
5074 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5075 kMaxPayloadLength);
5076 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5077
5078 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5079 // Frame should not be dropped.
5080 WaitForEncodedFrame(2);
5081
5082 // Trigger HD "singlecast"
5083 video_encoder_config.simulcast_layers[0].active = false;
5084 video_encoder_config.simulcast_layers[1].active = false;
5085 video_encoder_config.simulcast_layers[2].active = true;
5086
5087 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5088 kMaxPayloadLength);
5089 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5090
5091 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5092 // Frame should be dropped because of initial frame drop.
5093 ExpectDroppedFrame();
5094
5095 // Expect the sink_wants to specify a scaled frame.
5096 EXPECT_TRUE_WAIT(
5097 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5098 video_stream_encoder_->Stop();
5099}
5100
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005101TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5102 const int kLowTargetBitrateBps = 400000;
5103 // Set simulcast.
5104 ResetEncoder("VP9", 1, 1, 3, false);
5105 fake_encoder_.SetQualityScaling(true);
5106 const int kWidth = 1280;
5107 const int kHeight = 720;
5108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5109 DataRate::BitsPerSec(kLowTargetBitrateBps),
5110 DataRate::BitsPerSec(kLowTargetBitrateBps),
5111 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5112 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5113 // Frame should not be dropped.
5114 WaitForEncodedFrame(1);
5115
5116 // Trigger QVGA "singlecast"
5117 // Update the config.
5118 VideoEncoderConfig video_encoder_config;
5119 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5120 &video_encoder_config);
5121 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5122 vp9_settings.numberOfSpatialLayers = 3;
5123 // Since only one layer is active - automatic resize should be enabled.
5124 vp9_settings.automaticResizeOn = true;
5125 video_encoder_config.encoder_specific_settings =
5126 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5127 vp9_settings);
5128 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5129 video_encoder_config.content_type =
5130 VideoEncoderConfig::ContentType::kRealtimeVideo;
5131 // Currently simulcast layers |active| flags are used to inidicate
5132 // which SVC layers are active.
5133 video_encoder_config.simulcast_layers.resize(3);
5134
5135 video_encoder_config.simulcast_layers[0].active = true;
5136 video_encoder_config.simulcast_layers[1].active = false;
5137 video_encoder_config.simulcast_layers[2].active = false;
5138
5139 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5140 kMaxPayloadLength);
5141 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5142
5143 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5144 // Frame should not be dropped.
5145 WaitForEncodedFrame(2);
5146
5147 // Trigger HD "singlecast"
5148 video_encoder_config.simulcast_layers[0].active = false;
5149 video_encoder_config.simulcast_layers[1].active = false;
5150 video_encoder_config.simulcast_layers[2].active = true;
5151
5152 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5153 kMaxPayloadLength);
5154 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5155
5156 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5157 // Frame should be dropped because of initial frame drop.
5158 ExpectDroppedFrame();
5159
5160 // Expect the sink_wants to specify a scaled frame.
5161 EXPECT_TRUE_WAIT(
5162 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5163 video_stream_encoder_->Stop();
5164}
5165
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005166TEST_F(VideoStreamEncoderTest,
5167 InitialFrameDropActivatesWhenResolutionIncreases) {
5168 const int kWidth = 640;
5169 const int kHeight = 360;
5170
5171 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5172 DataRate::BitsPerSec(kTargetBitrateBps),
5173 DataRate::BitsPerSec(kTargetBitrateBps),
5174 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5175 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5176 // Frame should not be dropped.
5177 WaitForEncodedFrame(1);
5178
5179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5180 DataRate::BitsPerSec(kLowTargetBitrateBps),
5181 DataRate::BitsPerSec(kLowTargetBitrateBps),
5182 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5183 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5184 // Frame should not be dropped, bitrate not too low for frame.
5185 WaitForEncodedFrame(2);
5186
5187 // Incoming resolution increases.
5188 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5189 // Expect to drop this frame, bitrate too low for frame.
5190 ExpectDroppedFrame();
5191
5192 // Expect the sink_wants to specify a scaled frame.
5193 EXPECT_TRUE_WAIT(
5194 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5195 video_stream_encoder_->Stop();
5196}
5197
5198TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5199 const int kWidth = 640;
5200 const int kHeight = 360;
5201 // So that quality scaling doesn't happen by itself.
5202 fake_encoder_.SetQp(kQpHigh);
5203
5204 AdaptingFrameForwarder source(&time_controller_);
5205 source.set_adaptation_enabled(true);
5206 video_stream_encoder_->SetSource(
5207 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5208
5209 int timestamp = 1;
5210
5211 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5212 DataRate::BitsPerSec(kTargetBitrateBps),
5213 DataRate::BitsPerSec(kTargetBitrateBps),
5214 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5215 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5216 WaitForEncodedFrame(timestamp);
5217 timestamp += 9000;
5218 // Long pause to disable all first BWE drop logic.
5219 AdvanceTime(TimeDelta::Millis(1000));
5220
5221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5222 DataRate::BitsPerSec(kLowTargetBitrateBps),
5223 DataRate::BitsPerSec(kLowTargetBitrateBps),
5224 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5225 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5226 // Not dropped frame, as initial frame drop is disabled by now.
5227 WaitForEncodedFrame(timestamp);
5228 timestamp += 9000;
5229 AdvanceTime(TimeDelta::Millis(100));
5230
5231 // Quality adaptation down.
5232 video_stream_encoder_->TriggerQualityLow();
5233
5234 // Adaptation has an effect.
5235 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5236 5000);
5237
5238 // Frame isn't dropped as initial frame dropper is disabled.
5239 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5240 WaitForEncodedFrame(timestamp);
5241 timestamp += 9000;
5242 AdvanceTime(TimeDelta::Millis(100));
5243
5244 // Quality adaptation up.
5245 video_stream_encoder_->TriggerQualityHigh();
5246
5247 // Adaptation has an effect.
5248 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5249 5000);
5250
5251 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5252 // Frame should not be dropped, as initial framedropper is off.
5253 WaitForEncodedFrame(timestamp);
5254
5255 video_stream_encoder_->Stop();
5256}
5257
Åsa Perssone644a032019-11-08 15:56:00 +01005258TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5259 webrtc::test::ScopedFieldTrials field_trials(
5260 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5261
5262 // Reset encoder for field trials to take effect.
5263 VideoEncoderConfig config = video_encoder_config_.Copy();
5264 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005265 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005266 ConfigureEncoder(std::move(config));
5267 fake_encoder_.SetQp(kQpLow);
5268
5269 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005270 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005271 source.set_adaptation_enabled(true);
5272 video_stream_encoder_->SetSource(&source,
5273 DegradationPreference::MAINTAIN_FRAMERATE);
5274
5275 // Start at low bitrate.
5276 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5278 DataRate::BitsPerSec(kLowBitrateBps),
5279 DataRate::BitsPerSec(kLowBitrateBps),
5280 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005281
5282 // Expect first frame to be dropped and resolution to be limited.
5283 const int kWidth = 1280;
5284 const int kHeight = 720;
5285 const int64_t kFrameIntervalMs = 100;
5286 int64_t timestamp_ms = kFrameIntervalMs;
5287 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5288 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005289 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5290 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005291
5292 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5294 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005295
5296 // Insert frames and advance |min_duration_ms|.
5297 for (size_t i = 1; i <= 10; i++) {
5298 timestamp_ms += kFrameIntervalMs;
5299 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5300 WaitForEncodedFrame(timestamp_ms);
5301 }
5302 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5303 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5304
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005305 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005306
5307 // Insert frame should trigger high BW and release quality limitation.
5308 timestamp_ms += kFrameIntervalMs;
5309 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5310 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005311 // The ramp-up code involves the adaptation queue, give it time to execute.
5312 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005314 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005315
5316 // Frame should not be adapted.
5317 timestamp_ms += kFrameIntervalMs;
5318 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5319 WaitForEncodedFrame(kWidth, kHeight);
5320 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5321
5322 video_stream_encoder_->Stop();
5323}
5324
mflodmancc3d4422017-08-03 08:27:51 -07005325TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005326 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005327 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005328 source.set_adaptation_enabled(true);
5329 video_stream_encoder_->SetSource(&source,
5330 DegradationPreference::MAINTAIN_FRAMERATE);
5331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5332 DataRate::BitsPerSec(kTargetBitrateBps),
5333 DataRate::BitsPerSec(kTargetBitrateBps),
5334 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5335 fake_encoder_.SetQp(kQpHigh + 1);
5336 const int kWidth = 1280;
5337 const int kHeight = 720;
5338 const int64_t kFrameIntervalMs = 100;
5339 int64_t timestamp_ms = kFrameIntervalMs;
5340 for (size_t i = 1; i <= 100; i++) {
5341 timestamp_ms += kFrameIntervalMs;
5342 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5343 WaitForEncodedFrame(timestamp_ms);
5344 }
5345 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5346 // for the first time.
5347 // TODO(eshr): We should avoid these waits by using threads with simulated
5348 // time.
5349 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5350 2000 * 2.5 * 2);
5351 timestamp_ms += kFrameIntervalMs;
5352 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5353 WaitForEncodedFrame(timestamp_ms);
5354 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5355 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5356 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5357
5358 // Disable Quality scaling by turning off scaler on the encoder and
5359 // reconfiguring.
5360 fake_encoder_.SetQualityScaling(false);
5361 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5362 kMaxPayloadLength);
5363 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005364 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005365 // Since we turned off the quality scaler, the adaptations made by it are
5366 // removed.
5367 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5368 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5369
5370 video_stream_encoder_->Stop();
5371}
5372
5373TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005374 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5375 const int kTooSmallWidth = 10;
5376 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005377 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005378 DataRate::BitsPerSec(kTargetBitrateBps),
5379 DataRate::BitsPerSec(kTargetBitrateBps),
5380 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005381
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005382 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005383 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005384 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005385 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005386 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005387 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5388
5389 // Trigger adapt down, too small frame, expect no change.
5390 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005391 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005392 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005393 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005394 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5395 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5396
mflodmancc3d4422017-08-03 08:27:51 -07005397 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005398}
5399
mflodmancc3d4422017-08-03 08:27:51 -07005400TEST_F(VideoStreamEncoderTest,
5401 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005402 const int kTooSmallWidth = 10;
5403 const int kTooSmallHeight = 10;
5404 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005405 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005406 DataRate::BitsPerSec(kTargetBitrateBps),
5407 DataRate::BitsPerSec(kTargetBitrateBps),
5408 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005409
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005410 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07005411 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005412 video_stream_encoder_->SetSource(&source,
5413 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005414 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07005415 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5416 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5417
5418 // Trigger adapt down, expect limited framerate.
5419 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005420 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005421 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005422 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5426
5427 // Trigger adapt down, too small frame, expect no change.
5428 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005429 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07005430 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005431 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005432 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5434 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5435
mflodmancc3d4422017-08-03 08:27:51 -07005436 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005437}
5438
mflodmancc3d4422017-08-03 08:27:51 -07005439TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07005440 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02005441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005442 DataRate::BitsPerSec(kTargetBitrateBps),
5443 DataRate::BitsPerSec(kTargetBitrateBps),
5444 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02005445 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07005446 const int kFrameWidth = 1280;
5447 const int kFrameHeight = 720;
5448 video_source_.IncomingCapturedFrame(
5449 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005450 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07005451 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07005452}
5453
sprangb1ca0732017-02-01 08:38:12 -08005454// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07005455TEST_F(VideoStreamEncoderTest,
5456 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02005457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005458 DataRate::BitsPerSec(kTargetBitrateBps),
5459 DataRate::BitsPerSec(kTargetBitrateBps),
5460 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08005461
5462 const int kFrameWidth = 1280;
5463 const int kFrameHeight = 720;
5464 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07005465 // requested by
5466 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08005467 video_source_.set_adaptation_enabled(true);
5468
5469 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005470 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005471 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005472
5473 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005474 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08005475 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005476 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005477 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08005478
asaperssonfab67072017-04-04 05:51:49 -07005479 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02005480 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08005481 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005482 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005483 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005484
mflodmancc3d4422017-08-03 08:27:51 -07005485 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08005486}
sprangfe627f32017-03-29 08:24:59 -07005487
mflodmancc3d4422017-08-03 08:27:51 -07005488TEST_F(VideoStreamEncoderTest,
5489 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07005490 const int kFrameWidth = 1280;
5491 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07005492
Henrik Boström381d1092020-05-12 18:49:07 +02005493 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005494 DataRate::BitsPerSec(kTargetBitrateBps),
5495 DataRate::BitsPerSec(kTargetBitrateBps),
5496 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005497 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005498 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005499 video_source_.set_adaptation_enabled(true);
5500
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005501 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005502
5503 video_source_.IncomingCapturedFrame(
5504 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005505 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005506
5507 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07005508 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005509
5510 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07005511 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005512 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005513 video_source_.IncomingCapturedFrame(
5514 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005515 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005516 }
5517
5518 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07005519 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005520 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005521 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005522 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005523 video_source_.IncomingCapturedFrame(
5524 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005525 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005526 ++num_frames_dropped;
5527 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005528 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005529 }
5530 }
5531
sprang4847ae62017-06-27 07:06:52 -07005532 // Add some slack to account for frames dropped by the frame dropper.
5533 const int kErrorMargin = 1;
5534 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005535 kErrorMargin);
5536
5537 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07005538 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005539 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02005540 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005541 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005542 video_source_.IncomingCapturedFrame(
5543 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005544 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005545 ++num_frames_dropped;
5546 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005547 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005548 }
5549 }
sprang4847ae62017-06-27 07:06:52 -07005550 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07005551 kErrorMargin);
5552
5553 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02005554 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005555 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005556 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005557 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005558 video_source_.IncomingCapturedFrame(
5559 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005560 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005561 ++num_frames_dropped;
5562 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005563 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005564 }
5565 }
sprang4847ae62017-06-27 07:06:52 -07005566 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005567 kErrorMargin);
5568
5569 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02005570 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005571 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005572 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005573 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005574 video_source_.IncomingCapturedFrame(
5575 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005576 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005577 ++num_frames_dropped;
5578 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005579 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005580 }
5581 }
5582 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
5583
mflodmancc3d4422017-08-03 08:27:51 -07005584 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005585}
5586
mflodmancc3d4422017-08-03 08:27:51 -07005587TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07005588 const int kFramerateFps = 5;
5589 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07005590 const int kFrameWidth = 1280;
5591 const int kFrameHeight = 720;
5592
sprang4847ae62017-06-27 07:06:52 -07005593 // Reconfigure encoder with two temporal layers and screensharing, which will
5594 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02005595 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07005596
Henrik Boström381d1092020-05-12 18:49:07 +02005597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005598 DataRate::BitsPerSec(kTargetBitrateBps),
5599 DataRate::BitsPerSec(kTargetBitrateBps),
5600 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005601 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005602 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005603 video_source_.set_adaptation_enabled(true);
5604
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005605 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005606
5607 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08005608 rtc::VideoSinkWants last_wants;
5609 do {
5610 last_wants = video_source_.sink_wants();
5611
sprangc5d62e22017-04-02 23:53:04 -07005612 // Insert frames to get a new fps estimate...
5613 for (int j = 0; j < kFramerateFps; ++j) {
5614 video_source_.IncomingCapturedFrame(
5615 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08005616 if (video_source_.last_sent_width()) {
5617 sink_.WaitForEncodedFrame(timestamp_ms);
5618 }
sprangc5d62e22017-04-02 23:53:04 -07005619 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005620 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07005621 }
5622 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07005623 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08005624 } while (video_source_.sink_wants().max_framerate_fps <
5625 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07005626
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005627 EXPECT_THAT(video_source_.sink_wants(),
5628 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07005629
mflodmancc3d4422017-08-03 08:27:51 -07005630 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005631}
asaperssonf7e294d2017-06-13 23:25:22 -07005632
mflodmancc3d4422017-08-03 08:27:51 -07005633TEST_F(VideoStreamEncoderTest,
5634 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005635 const int kWidth = 1280;
5636 const int kHeight = 720;
5637 const int64_t kFrameIntervalMs = 150;
5638 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005640 DataRate::BitsPerSec(kTargetBitrateBps),
5641 DataRate::BitsPerSec(kTargetBitrateBps),
5642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005643
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005644 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005645 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005646 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005647 video_stream_encoder_->SetSource(&source,
5648 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005649 timestamp_ms += kFrameIntervalMs;
5650 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005651 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005652 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5655 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5656
5657 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005658 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005659 timestamp_ms += kFrameIntervalMs;
5660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005661 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005662 EXPECT_THAT(source.sink_wants(),
5663 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005664 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5665 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5666 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5667
5668 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005669 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005670 timestamp_ms += kFrameIntervalMs;
5671 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005672 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005673 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005674 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5676 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5677
5678 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005679 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005680 timestamp_ms += kFrameIntervalMs;
5681 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005682 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005683 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005684 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5685 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5686 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5687
5688 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005689 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005690 timestamp_ms += kFrameIntervalMs;
5691 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005692 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005693 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005694 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5695 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5696 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5697
5698 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005699 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005700 timestamp_ms += kFrameIntervalMs;
5701 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005702 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005703 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005704 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5705 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5706 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5707
5708 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005709 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005710 timestamp_ms += kFrameIntervalMs;
5711 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005712 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005713 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005714 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5715 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5716 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5717
5718 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07005719 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005720 timestamp_ms += kFrameIntervalMs;
5721 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005722 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005723 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005724 rtc::VideoSinkWants last_wants = source.sink_wants();
5725 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5726 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5727 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5728
5729 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005730 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005731 timestamp_ms += kFrameIntervalMs;
5732 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005733 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005734 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07005735 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5736 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5737 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5738
Evan Shrubsole64469032020-06-11 10:45:29 +02005739 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005740 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005741 timestamp_ms += kFrameIntervalMs;
5742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005743 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005744 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005745 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5746 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5747 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5748
5749 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005750 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005751 timestamp_ms += kFrameIntervalMs;
5752 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005753 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005754 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005755 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5756 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5757 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5758
5759 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005760 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005761 timestamp_ms += kFrameIntervalMs;
5762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005763 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005764 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005765 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5766 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5767 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5768
5769 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005770 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005771 timestamp_ms += kFrameIntervalMs;
5772 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005773 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005774 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005775 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5776 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5777 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5778
5779 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005780 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005781 timestamp_ms += kFrameIntervalMs;
5782 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005783 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005784 EXPECT_THAT(source.sink_wants(), FpsMax());
5785 EXPECT_EQ(source.sink_wants().max_pixel_count,
5786 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07005787 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5788 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5789 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5790
5791 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005792 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005793 timestamp_ms += kFrameIntervalMs;
5794 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005795 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005796 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005797 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5798 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5799 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5800
Åsa Persson30ab0152019-08-27 12:22:33 +02005801 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005802 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005803 timestamp_ms += kFrameIntervalMs;
5804 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005805 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005806 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005807 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005808 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5810 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5811
5812 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005813 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005815 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5816
mflodmancc3d4422017-08-03 08:27:51 -07005817 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005818}
5819
mflodmancc3d4422017-08-03 08:27:51 -07005820TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07005821 const int kWidth = 1280;
5822 const int kHeight = 720;
5823 const int64_t kFrameIntervalMs = 150;
5824 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005826 DataRate::BitsPerSec(kTargetBitrateBps),
5827 DataRate::BitsPerSec(kTargetBitrateBps),
5828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005829
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005830 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005831 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005832 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005833 video_stream_encoder_->SetSource(&source,
5834 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005835 timestamp_ms += kFrameIntervalMs;
5836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005837 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005838 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005839 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5841 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5843 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5845
5846 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005847 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005848 timestamp_ms += kFrameIntervalMs;
5849 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005850 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005851 EXPECT_THAT(source.sink_wants(),
5852 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5854 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5855 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5856 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5857 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5858 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5859
5860 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005861 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005862 timestamp_ms += kFrameIntervalMs;
5863 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005864 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005865 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005866 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5867 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5868 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5869 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5870 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5871 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5872
5873 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005874 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005875 timestamp_ms += kFrameIntervalMs;
5876 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005877 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005878 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02005879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07005880 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5881 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5883 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5884 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5885
Evan Shrubsole64469032020-06-11 10:45:29 +02005886 // Trigger cpu adapt up, expect no change since QP is most limited.
5887 {
5888 // Store current sink wants since we expect no change and if there is no
5889 // change then last_wants() is not updated.
5890 auto previous_sink_wants = source.sink_wants();
5891 video_stream_encoder_->TriggerCpuUnderuse();
5892 timestamp_ms += kFrameIntervalMs;
5893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5894 WaitForEncodedFrame(timestamp_ms);
5895 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
5896 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5897 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5898 }
5899
5900 // Trigger quality adapt up, expect increased fps (640x360@30fps).
5901 video_stream_encoder_->TriggerQualityHigh();
5902 timestamp_ms += kFrameIntervalMs;
5903 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5904 WaitForEncodedFrame(timestamp_ms);
5905 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
5906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5907 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5908 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5909 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5910 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5911 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5912
5913 // Trigger quality adapt up and Cpu adapt up since both are most limited,
5914 // expect increased resolution (960x540@30fps).
5915 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02005916 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005917 timestamp_ms += kFrameIntervalMs;
5918 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005919 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02005920 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005921 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5922 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5923 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5924 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5925 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005926 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005927
Evan Shrubsole64469032020-06-11 10:45:29 +02005928 // Trigger quality adapt up and Cpu adapt up since both are most limited,
5929 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005930 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02005931 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005932 timestamp_ms += kFrameIntervalMs;
5933 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005934 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005935 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005936 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005937 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5938 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5939 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5940 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5941 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005942 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005943
5944 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005945 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005946 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005947 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005948 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005949
mflodmancc3d4422017-08-03 08:27:51 -07005950 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005951}
5952
mflodmancc3d4422017-08-03 08:27:51 -07005953TEST_F(VideoStreamEncoderTest,
5954 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07005955 const int kWidth = 640;
5956 const int kHeight = 360;
5957 const int kFpsLimit = 15;
5958 const int64_t kFrameIntervalMs = 150;
5959 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005961 DataRate::BitsPerSec(kTargetBitrateBps),
5962 DataRate::BitsPerSec(kTargetBitrateBps),
5963 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005964
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005965 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005966 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005967 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005968 video_stream_encoder_->SetSource(&source,
5969 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005970 timestamp_ms += kFrameIntervalMs;
5971 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005972 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005973 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005974 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5975 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5976 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5977 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5978 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5979 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5980
5981 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005982 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005983 timestamp_ms += kFrameIntervalMs;
5984 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005985 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005986 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005987 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5988 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5989 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5990 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5991 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5992 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5993
5994 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005995 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005996 timestamp_ms += kFrameIntervalMs;
5997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005998 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005999 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006000 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006001 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006002 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6003 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6004 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6005 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6006
Evan Shrubsole64469032020-06-11 10:45:29 +02006007 // Trigger cpu adapt up, expect no change because quality is most limited.
6008 {
6009 auto previous_sink_wants = source.sink_wants();
6010 // Store current sink wants since we expect no change ind if there is no
6011 // change then last__wants() is not updated.
6012 video_stream_encoder_->TriggerCpuUnderuse();
6013 timestamp_ms += kFrameIntervalMs;
6014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6015 WaitForEncodedFrame(timestamp_ms);
6016 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6017 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6018 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6019 }
6020
6021 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6022 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006023 timestamp_ms += kFrameIntervalMs;
6024 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006025 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006026 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006027 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6028 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6029 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006030 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6031 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6032 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006033
Evan Shrubsole64469032020-06-11 10:45:29 +02006034 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006035 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006036 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006037 timestamp_ms += kFrameIntervalMs;
6038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006039 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006040 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6042 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6043 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6044 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6045 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006046 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006047
6048 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006049 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006051 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006052 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006053
mflodmancc3d4422017-08-03 08:27:51 -07006054 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006055}
6056
mflodmancc3d4422017-08-03 08:27:51 -07006057TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006058 const int kFrameWidth = 1920;
6059 const int kFrameHeight = 1080;
6060 // 3/4 of 1920.
6061 const int kAdaptedFrameWidth = 1440;
6062 // 3/4 of 1080 rounded down to multiple of 4.
6063 const int kAdaptedFrameHeight = 808;
6064 const int kFramerate = 24;
6065
Henrik Boström381d1092020-05-12 18:49:07 +02006066 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006067 DataRate::BitsPerSec(kTargetBitrateBps),
6068 DataRate::BitsPerSec(kTargetBitrateBps),
6069 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006070 // Trigger reconfigure encoder (without resetting the entire instance).
6071 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006072 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6073 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006074 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006075 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006076 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006077 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006078 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006079 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006080
6081 video_source_.set_adaptation_enabled(true);
6082
6083 video_source_.IncomingCapturedFrame(
6084 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006085 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006086
6087 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006088 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006089 video_source_.IncomingCapturedFrame(
6090 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006091 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006092
mflodmancc3d4422017-08-03 08:27:51 -07006093 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006094}
6095
mflodmancc3d4422017-08-03 08:27:51 -07006096TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006097 const int kFrameWidth = 1280;
6098 const int kFrameHeight = 720;
6099 const int kLowFps = 2;
6100 const int kHighFps = 30;
6101
Henrik Boström381d1092020-05-12 18:49:07 +02006102 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006103 DataRate::BitsPerSec(kTargetBitrateBps),
6104 DataRate::BitsPerSec(kTargetBitrateBps),
6105 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006106
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006107 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006108 max_framerate_ = kLowFps;
6109
6110 // Insert 2 seconds of 2fps video.
6111 for (int i = 0; i < kLowFps * 2; ++i) {
6112 video_source_.IncomingCapturedFrame(
6113 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6114 WaitForEncodedFrame(timestamp_ms);
6115 timestamp_ms += 1000 / kLowFps;
6116 }
6117
6118 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006119 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006120 DataRate::BitsPerSec(kTargetBitrateBps),
6121 DataRate::BitsPerSec(kTargetBitrateBps),
6122 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006123 video_source_.IncomingCapturedFrame(
6124 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6125 WaitForEncodedFrame(timestamp_ms);
6126 timestamp_ms += 1000 / kLowFps;
6127
6128 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6129
6130 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006131 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006132 const int kFrameIntervalMs = 1000 / kHighFps;
6133 max_framerate_ = kHighFps;
6134 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6135 video_source_.IncomingCapturedFrame(
6136 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6137 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6138 // be dropped if the encoder hans't been updated with the new higher target
6139 // framerate yet, causing it to overshoot the target bitrate and then
6140 // suffering the wrath of the media optimizer.
6141 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6142 timestamp_ms += kFrameIntervalMs;
6143 }
6144
6145 // Don expect correct measurement just yet, but it should be higher than
6146 // before.
6147 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6148
mflodmancc3d4422017-08-03 08:27:51 -07006149 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006150}
6151
mflodmancc3d4422017-08-03 08:27:51 -07006152TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006153 const int kFrameWidth = 1280;
6154 const int kFrameHeight = 720;
6155 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006156 ResetEncoder("FAKE", 1, 1, 1, false,
6157 VideoStreamEncoderSettings::BitrateAllocationCallbackType::
6158 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006159
Henrik Boström381d1092020-05-12 18:49:07 +02006160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006161 DataRate::BitsPerSec(kTargetBitrateBps),
6162 DataRate::BitsPerSec(kTargetBitrateBps),
6163 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006164 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006165
6166 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006167 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006168 video_source_.IncomingCapturedFrame(
6169 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6170 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006171 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006172
6173 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6175 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6176 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006177
6178 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006179 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006180 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006181
Per Kjellanderdcef6412020-10-07 15:09:05 +02006182 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006183 video_source_.IncomingCapturedFrame(
6184 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6185 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006186 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006187
mflodmancc3d4422017-08-03 08:27:51 -07006188 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006189}
ilnik6b826ef2017-06-16 06:53:48 -07006190
Niels Möller4db138e2018-04-19 09:04:13 +02006191TEST_F(VideoStreamEncoderTest,
6192 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6193 const int kFrameWidth = 1280;
6194 const int kFrameHeight = 720;
6195 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006196 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006197 DataRate::BitsPerSec(kTargetBitrateBps),
6198 DataRate::BitsPerSec(kTargetBitrateBps),
6199 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006200 video_source_.IncomingCapturedFrame(
6201 CreateFrame(1, kFrameWidth, kFrameHeight));
6202 WaitForEncodedFrame(1);
6203 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6204 .low_encode_usage_threshold_percent,
6205 default_options.low_encode_usage_threshold_percent);
6206 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6207 .high_encode_usage_threshold_percent,
6208 default_options.high_encode_usage_threshold_percent);
6209 video_stream_encoder_->Stop();
6210}
6211
6212TEST_F(VideoStreamEncoderTest,
6213 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6214 const int kFrameWidth = 1280;
6215 const int kFrameHeight = 720;
6216 CpuOveruseOptions hardware_options;
6217 hardware_options.low_encode_usage_threshold_percent = 150;
6218 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006219 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006220
Henrik Boström381d1092020-05-12 18:49:07 +02006221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006222 DataRate::BitsPerSec(kTargetBitrateBps),
6223 DataRate::BitsPerSec(kTargetBitrateBps),
6224 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006225 video_source_.IncomingCapturedFrame(
6226 CreateFrame(1, kFrameWidth, kFrameHeight));
6227 WaitForEncodedFrame(1);
6228 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6229 .low_encode_usage_threshold_percent,
6230 hardware_options.low_encode_usage_threshold_percent);
6231 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6232 .high_encode_usage_threshold_percent,
6233 hardware_options.high_encode_usage_threshold_percent);
6234 video_stream_encoder_->Stop();
6235}
6236
Niels Möller6bb5ab92019-01-11 11:11:10 +01006237TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6238 const int kFrameWidth = 320;
6239 const int kFrameHeight = 240;
6240 const int kFps = 30;
6241 const int kTargetBitrateBps = 120000;
6242 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6243
Henrik Boström381d1092020-05-12 18:49:07 +02006244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006245 DataRate::BitsPerSec(kTargetBitrateBps),
6246 DataRate::BitsPerSec(kTargetBitrateBps),
6247 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006248
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006249 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006250 max_framerate_ = kFps;
6251
6252 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6253 fake_encoder_.SimulateOvershoot(1.0);
6254 int num_dropped = 0;
6255 for (int i = 0; i < kNumFramesInRun; ++i) {
6256 video_source_.IncomingCapturedFrame(
6257 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6258 // Wait up to two frame durations for a frame to arrive.
6259 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6260 ++num_dropped;
6261 }
6262 timestamp_ms += 1000 / kFps;
6263 }
6264
Erik Språnga8d48ab2019-02-08 14:17:40 +01006265 // Framerate should be measured to be near the expected target rate.
6266 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6267
6268 // Frame drops should be within 5% of expected 0%.
6269 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006270
6271 // Make encoder produce frames at double the expected bitrate during 3 seconds
6272 // of video, verify number of drops. Rate needs to be slightly changed in
6273 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006274 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006275 const RateControlSettings trials =
6276 RateControlSettings::ParseFromFieldTrials();
6277 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006278 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006279 // frame dropping since the adjuter will try to just lower the target
6280 // bitrate rather than drop frames. If network headroom can be used, it
6281 // doesn't push back as hard so we don't need quite as much overshoot.
6282 // These numbers are unfortunately a bit magical but there's not trivial
6283 // way to algebraically infer them.
6284 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6285 overshoot_factor = 2.4;
6286 } else {
6287 overshoot_factor = 4.0;
6288 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006289 }
6290 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006292 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6293 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6294 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006295 num_dropped = 0;
6296 for (int i = 0; i < kNumFramesInRun; ++i) {
6297 video_source_.IncomingCapturedFrame(
6298 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6299 // Wait up to two frame durations for a frame to arrive.
6300 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6301 ++num_dropped;
6302 }
6303 timestamp_ms += 1000 / kFps;
6304 }
6305
Henrik Boström381d1092020-05-12 18:49:07 +02006306 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006307 DataRate::BitsPerSec(kTargetBitrateBps),
6308 DataRate::BitsPerSec(kTargetBitrateBps),
6309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006310
6311 // Target framerate should be still be near the expected target, despite
6312 // the frame drops.
6313 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6314
6315 // Frame drops should be within 5% of expected 50%.
6316 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006317
6318 video_stream_encoder_->Stop();
6319}
6320
6321TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6322 const int kFrameWidth = 320;
6323 const int kFrameHeight = 240;
6324 const int kActualInputFps = 24;
6325 const int kTargetBitrateBps = 120000;
6326
6327 ASSERT_GT(max_framerate_, kActualInputFps);
6328
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006329 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006330 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006332 DataRate::BitsPerSec(kTargetBitrateBps),
6333 DataRate::BitsPerSec(kTargetBitrateBps),
6334 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006335
6336 // Insert 3 seconds of video, with an input fps lower than configured max.
6337 for (int i = 0; i < kActualInputFps * 3; ++i) {
6338 video_source_.IncomingCapturedFrame(
6339 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6340 // Wait up to two frame durations for a frame to arrive.
6341 WaitForEncodedFrame(timestamp_ms);
6342 timestamp_ms += 1000 / kActualInputFps;
6343 }
6344
6345 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6346
6347 video_stream_encoder_->Stop();
6348}
6349
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006350TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006351 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006353 DataRate::BitsPerSec(kTargetBitrateBps),
6354 DataRate::BitsPerSec(kTargetBitrateBps),
6355 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006356
6357 fake_encoder_.BlockNextEncode();
6358 video_source_.IncomingCapturedFrame(
6359 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6360 WaitForEncodedFrame(1);
6361 // On the very first frame full update should be forced.
6362 rect = fake_encoder_.GetLastUpdateRect();
6363 EXPECT_EQ(rect.offset_x, 0);
6364 EXPECT_EQ(rect.offset_y, 0);
6365 EXPECT_EQ(rect.height, codec_height_);
6366 EXPECT_EQ(rect.width, codec_width_);
6367 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6368 // call to ContinueEncode.
6369 video_source_.IncomingCapturedFrame(
6370 CreateFrameWithUpdatedPixel(2, nullptr, 1));
6371 ExpectDroppedFrame();
6372 video_source_.IncomingCapturedFrame(
6373 CreateFrameWithUpdatedPixel(3, nullptr, 10));
6374 ExpectDroppedFrame();
6375 fake_encoder_.ContinueEncode();
6376 WaitForEncodedFrame(3);
6377 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
6378 rect = fake_encoder_.GetLastUpdateRect();
6379 EXPECT_EQ(rect.offset_x, 1);
6380 EXPECT_EQ(rect.offset_y, 0);
6381 EXPECT_EQ(rect.width, 10);
6382 EXPECT_EQ(rect.height, 1);
6383
6384 video_source_.IncomingCapturedFrame(
6385 CreateFrameWithUpdatedPixel(4, nullptr, 0));
6386 WaitForEncodedFrame(4);
6387 // Previous frame was encoded, so no accumulation should happen.
6388 rect = fake_encoder_.GetLastUpdateRect();
6389 EXPECT_EQ(rect.offset_x, 0);
6390 EXPECT_EQ(rect.offset_y, 0);
6391 EXPECT_EQ(rect.width, 1);
6392 EXPECT_EQ(rect.height, 1);
6393
6394 video_stream_encoder_->Stop();
6395}
6396
Erik Språngd7329ca2019-02-21 21:19:53 +01006397TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02006398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006399 DataRate::BitsPerSec(kTargetBitrateBps),
6400 DataRate::BitsPerSec(kTargetBitrateBps),
6401 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006402
6403 // First frame is always keyframe.
6404 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6405 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01006406 EXPECT_THAT(
6407 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006408 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006409
6410 // Insert delta frame.
6411 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6412 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01006413 EXPECT_THAT(
6414 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006415 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006416
6417 // Request next frame be a key-frame.
6418 video_stream_encoder_->SendKeyFrame();
6419 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6420 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01006421 EXPECT_THAT(
6422 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006423 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006424
6425 video_stream_encoder_->Stop();
6426}
6427
6428TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
6429 // Setup simulcast with three streams.
6430 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006431 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006432 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6433 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6434 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006435 // Wait for all three layers before triggering event.
6436 sink_.SetNumExpectedLayers(3);
6437
6438 // First frame is always keyframe.
6439 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6440 WaitForEncodedFrame(1);
6441 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006442 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6443 VideoFrameType::kVideoFrameKey,
6444 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006445
6446 // Insert delta frame.
6447 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6448 WaitForEncodedFrame(2);
6449 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006450 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6451 VideoFrameType::kVideoFrameDelta,
6452 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006453
6454 // Request next frame be a key-frame.
6455 // Only first stream is configured to produce key-frame.
6456 video_stream_encoder_->SendKeyFrame();
6457 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6458 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02006459
6460 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
6461 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01006462 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006463 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02006464 VideoFrameType::kVideoFrameKey,
6465 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006466
6467 video_stream_encoder_->Stop();
6468}
6469
6470TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
6471 // Configure internal source factory and setup test again.
6472 encoder_factory_.SetHasInternalSource(true);
6473 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006475 DataRate::BitsPerSec(kTargetBitrateBps),
6476 DataRate::BitsPerSec(kTargetBitrateBps),
6477 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006478
6479 // Call encoder directly, simulating internal source where encoded frame
6480 // callback in VideoStreamEncoder is called despite no OnFrame().
6481 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
6482 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006483 EXPECT_THAT(
6484 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006485 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006486
Niels Möller8f7ce222019-03-21 15:43:58 +01006487 const std::vector<VideoFrameType> kDeltaFrame = {
6488 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01006489 // Need to set timestamp manually since manually for injected frame.
6490 VideoFrame frame = CreateFrame(101, nullptr);
6491 frame.set_timestamp(101);
6492 fake_encoder_.InjectFrame(frame, false);
6493 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006494 EXPECT_THAT(
6495 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006496 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006497
6498 // Request key-frame. The forces a dummy frame down into the encoder.
6499 fake_encoder_.ExpectNullFrame();
6500 video_stream_encoder_->SendKeyFrame();
6501 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006502 EXPECT_THAT(
6503 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006504 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006505
6506 video_stream_encoder_->Stop();
6507}
Erik Språngb7cb7b52019-02-26 15:52:33 +01006508
6509TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
6510 // Configure internal source factory and setup test again.
6511 encoder_factory_.SetHasInternalSource(true);
6512 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006514 DataRate::BitsPerSec(kTargetBitrateBps),
6515 DataRate::BitsPerSec(kTargetBitrateBps),
6516 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006517
6518 int64_t timestamp = 1;
6519 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02006520 image.SetEncodedData(
6521 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01006522 image.capture_time_ms_ = ++timestamp;
6523 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
6524 const int64_t kEncodeFinishDelayMs = 10;
6525 image.timing_.encode_start_ms = timestamp;
6526 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
6527 fake_encoder_.InjectEncodedImage(image);
6528 // Wait for frame without incrementing clock.
6529 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
6530 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
6531 // capture timestamp should be kEncodeFinishDelayMs in the past.
6532 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006533 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006534
6535 video_stream_encoder_->Stop();
6536}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006537
6538TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006539 // SPS contains VUI with restrictions on the maximum number of reordered
6540 // pictures, there is no need to rewrite the bitstream to enable faster
6541 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006542 ResetEncoder("H264", 1, 1, 1, false);
6543
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6545 DataRate::BitsPerSec(kTargetBitrateBps),
6546 DataRate::BitsPerSec(kTargetBitrateBps),
6547 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6548 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006549
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006550 fake_encoder_.SetEncodedImageData(
6551 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006552
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006553 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6554 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006555
6556 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6557 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006558
6559 video_stream_encoder_->Stop();
6560}
6561
6562TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006563 // SPS does not contain VUI, the bitstream is will be rewritten with added
6564 // VUI with restrictions on the maximum number of reordered pictures to
6565 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006566 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
6567 0x00, 0x00, 0x03, 0x03, 0xF4,
6568 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006569 ResetEncoder("H264", 1, 1, 1, false);
6570
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6572 DataRate::BitsPerSec(kTargetBitrateBps),
6573 DataRate::BitsPerSec(kTargetBitrateBps),
6574 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006576
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006577 fake_encoder_.SetEncodedImageData(
6578 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006579
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006580 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6581 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006582
6583 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6584 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006585
6586 video_stream_encoder_->Stop();
6587}
6588
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006589TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
6590 const int kFrameWidth = 1280;
6591 const int kFrameHeight = 720;
6592 const int kTargetBitrateBps = 300000; // To low for HD resolution.
6593
Henrik Boström381d1092020-05-12 18:49:07 +02006594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006595 DataRate::BitsPerSec(kTargetBitrateBps),
6596 DataRate::BitsPerSec(kTargetBitrateBps),
6597 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006598 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6599
6600 // Insert a first video frame. It should be dropped because of downscale in
6601 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006602 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006603 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6604 frame.set_rotation(kVideoRotation_270);
6605 video_source_.IncomingCapturedFrame(frame);
6606
6607 ExpectDroppedFrame();
6608
6609 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006610 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006611 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6612 frame.set_rotation(kVideoRotation_90);
6613 video_source_.IncomingCapturedFrame(frame);
6614
6615 WaitForEncodedFrame(timestamp_ms);
6616 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
6617
6618 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006619 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006620 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6621 frame.set_rotation(kVideoRotation_180);
6622 video_source_.IncomingCapturedFrame(frame);
6623
6624 WaitForEncodedFrame(timestamp_ms);
6625 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
6626
6627 video_stream_encoder_->Stop();
6628}
6629
Erik Språng5056af02019-09-02 15:53:11 +02006630TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
6631 const int kFrameWidth = 320;
6632 const int kFrameHeight = 180;
6633
6634 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02006635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006636 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
6637 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
6638 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02006639 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006640 /*rtt_ms=*/0,
6641 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006642
6643 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006644 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02006645 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6646 frame.set_rotation(kVideoRotation_270);
6647 video_source_.IncomingCapturedFrame(frame);
6648 WaitForEncodedFrame(timestamp_ms);
6649
6650 // Set a target rate below the minimum allowed by the codec settings.
6651 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006652 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
6653 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02006654 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02006655 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02006656 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02006657 /*link_allocation=*/target_rate,
6658 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006659 /*rtt_ms=*/0,
6660 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006661 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6662
6663 // Target bitrate and bandwidth allocation should both be capped at min_rate.
6664 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6665 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006666 DataRate allocation_sum =
6667 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02006668 EXPECT_EQ(min_rate, allocation_sum);
6669 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
6670
6671 video_stream_encoder_->Stop();
6672}
6673
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006674TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02006675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006676 DataRate::BitsPerSec(kTargetBitrateBps),
6677 DataRate::BitsPerSec(kTargetBitrateBps),
6678 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006679 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006680 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006681 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6682 WaitForEncodedFrame(1);
6683
6684 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6685 ASSERT_TRUE(prev_rate_settings.has_value());
6686 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
6687 kDefaultFramerate);
6688
6689 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
6690 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
6691 timestamp_ms += 1000 / kDefaultFramerate;
6692 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6693 WaitForEncodedFrame(timestamp_ms);
6694 }
6695 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
6696 kDefaultFramerate);
6697 // Capture larger frame to trigger a reconfigure.
6698 codec_height_ *= 2;
6699 codec_width_ *= 2;
6700 timestamp_ms += 1000 / kDefaultFramerate;
6701 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6702 WaitForEncodedFrame(timestamp_ms);
6703
6704 EXPECT_EQ(2, sink_.number_of_reconfigurations());
6705 auto current_rate_settings =
6706 fake_encoder_.GetAndResetLastRateControlSettings();
6707 // Ensure we have actually reconfigured twice
6708 // The rate settings should have been set again even though
6709 // they haven't changed.
6710 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006711 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006712
6713 video_stream_encoder_->Stop();
6714}
6715
philipeld9cc8c02019-09-16 14:53:40 +02006716struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02006717 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
6718 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
6719 MOCK_METHOD(void,
6720 RequestEncoderSwitch,
6721 (const webrtc::SdpVideoFormat& format),
6722 (override));
philipeld9cc8c02019-09-16 14:53:40 +02006723};
6724
6725TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
6726 constexpr int kDontCare = 100;
6727
6728 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6729 video_send_config_.encoder_settings.encoder_switch_request_callback =
6730 &switch_callback;
6731 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6732 encoder_config.codec_type = kVideoCodecVP8;
6733 webrtc::test::ScopedFieldTrials field_trial(
6734 "WebRTC-NetworkCondition-EncoderSwitch/"
6735 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6736 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6737
6738 // Reset encoder for new configuration to take effect.
6739 ConfigureEncoder(std::move(encoder_config));
6740
6741 // Send one frame to trigger ReconfigureEncoder.
6742 video_source_.IncomingCapturedFrame(
6743 CreateFrame(kDontCare, kDontCare, kDontCare));
6744
6745 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006746 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6747 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006748 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006749 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006750
Henrik Boström381d1092020-05-12 18:49:07 +02006751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006752 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6753 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6754 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02006755 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006756 /*rtt_ms=*/0,
6757 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006758 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02006759
6760 video_stream_encoder_->Stop();
6761}
6762
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006763TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
6764 constexpr int kDontCare = 100;
6765
6766 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6767 video_send_config_.encoder_settings.encoder_switch_request_callback =
6768 &switch_callback;
6769 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6770 encoder_config.codec_type = kVideoCodecVP8;
6771 webrtc::test::ScopedFieldTrials field_trial(
6772 "WebRTC-NetworkCondition-EncoderSwitch/"
6773 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6774 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6775
6776 // Reset encoder for new configuration to take effect.
6777 ConfigureEncoder(std::move(encoder_config));
6778
6779 // Send one frame to trigger ReconfigureEncoder.
6780 video_source_.IncomingCapturedFrame(
6781 CreateFrame(kDontCare, kDontCare, kDontCare));
6782
6783 using Config = EncoderSwitchRequestCallback::Config;
6784 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
6785 .Times(0);
6786
Henrik Boström381d1092020-05-12 18:49:07 +02006787 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006788 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
6789 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
6790 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
6791 /*fraction_lost=*/0,
6792 /*rtt_ms=*/0,
6793 /*cwnd_reduce_ratio=*/0);
6794
6795 video_stream_encoder_->Stop();
6796}
6797
philipeld9cc8c02019-09-16 14:53:40 +02006798TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
6799 constexpr int kSufficientBitrateToNotDrop = 1000;
6800 constexpr int kHighRes = 500;
6801 constexpr int kLowRes = 100;
6802
6803 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6804 video_send_config_.encoder_settings.encoder_switch_request_callback =
6805 &switch_callback;
6806 webrtc::test::ScopedFieldTrials field_trial(
6807 "WebRTC-NetworkCondition-EncoderSwitch/"
6808 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
6809 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6810 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6811 encoder_config.codec_type = kVideoCodecH264;
6812
6813 // Reset encoder for new configuration to take effect.
6814 ConfigureEncoder(std::move(encoder_config));
6815
6816 // The VideoStreamEncoder needs some bitrate before it can start encoding,
6817 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
6818 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02006819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006820 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6821 /*stable_target_bitrate=*/
6822 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6823 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02006824 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006825 /*rtt_ms=*/0,
6826 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02006827
6828 // Send one frame to trigger ReconfigureEncoder.
6829 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
6830 WaitForEncodedFrame(1);
6831
6832 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006833 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6834 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006835 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006836 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006837
6838 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
6839 WaitForEncodedFrame(2);
6840
6841 video_stream_encoder_->Stop();
6842}
6843
philipel9b058032020-02-10 11:30:00 +01006844TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
6845 constexpr int kDontCare = 100;
6846 StrictMock<MockEncoderSelector> encoder_selector;
6847 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6848 &fake_encoder_, &encoder_selector);
6849 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6850
6851 // Reset encoder for new configuration to take effect.
6852 ConfigureEncoder(video_encoder_config_.Copy());
6853
6854 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
6855
6856 video_source_.IncomingCapturedFrame(
6857 CreateFrame(kDontCare, kDontCare, kDontCare));
6858 video_stream_encoder_->Stop();
6859
6860 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
6861 // to it's factory, so in order for the encoder instance in the
6862 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
6863 // reset the |video_stream_encoder_| here.
6864 video_stream_encoder_.reset();
6865}
6866
6867TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
6868 constexpr int kDontCare = 100;
6869
6870 NiceMock<MockEncoderSelector> encoder_selector;
6871 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6872 video_send_config_.encoder_settings.encoder_switch_request_callback =
6873 &switch_callback;
6874 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6875 &fake_encoder_, &encoder_selector);
6876 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6877
6878 // Reset encoder for new configuration to take effect.
6879 ConfigureEncoder(video_encoder_config_.Copy());
6880
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01006881 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01006882 .WillByDefault(Return(SdpVideoFormat("AV1")));
6883 EXPECT_CALL(switch_callback,
6884 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
6885 Field(&SdpVideoFormat::name, "AV1"))));
6886
Henrik Boström381d1092020-05-12 18:49:07 +02006887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006888 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6889 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6890 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01006891 /*fraction_lost=*/0,
6892 /*rtt_ms=*/0,
6893 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006894 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01006895
6896 video_stream_encoder_->Stop();
6897}
6898
6899TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
6900 constexpr int kSufficientBitrateToNotDrop = 1000;
6901 constexpr int kDontCare = 100;
6902
6903 NiceMock<MockVideoEncoder> video_encoder;
6904 NiceMock<MockEncoderSelector> encoder_selector;
6905 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6906 video_send_config_.encoder_settings.encoder_switch_request_callback =
6907 &switch_callback;
6908 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6909 &video_encoder, &encoder_selector);
6910 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6911
6912 // Reset encoder for new configuration to take effect.
6913 ConfigureEncoder(video_encoder_config_.Copy());
6914
6915 // The VideoStreamEncoder needs some bitrate before it can start encoding,
6916 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
6917 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02006918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006919 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6920 /*stable_target_bitrate=*/
6921 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6922 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01006923 /*fraction_lost=*/0,
6924 /*rtt_ms=*/0,
6925 /*cwnd_reduce_ratio=*/0);
6926
6927 ON_CALL(video_encoder, Encode(_, _))
6928 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
6929 ON_CALL(encoder_selector, OnEncoderBroken())
6930 .WillByDefault(Return(SdpVideoFormat("AV2")));
6931
6932 rtc::Event encode_attempted;
6933 EXPECT_CALL(switch_callback,
6934 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
6935 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
6936 EXPECT_EQ(format.name, "AV2");
6937 encode_attempted.Set();
6938 });
6939
6940 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
6941 encode_attempted.Wait(3000);
6942
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006943 AdvanceTime(TimeDelta::Millis(0));
6944
philipel9b058032020-02-10 11:30:00 +01006945 video_stream_encoder_->Stop();
6946
6947 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
6948 // to it's factory, so in order for the encoder instance in the
6949 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
6950 // reset the |video_stream_encoder_| here.
6951 video_stream_encoder_.reset();
6952}
6953
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006954TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006955 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006956 const int kFrameWidth = 320;
6957 const int kFrameHeight = 180;
6958
6959 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006960 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006962 /*target_bitrate=*/rate,
6963 /*stable_target_bitrate=*/rate,
6964 /*link_allocation=*/rate,
6965 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006966 /*rtt_ms=*/0,
6967 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006968
6969 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006970 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006971 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6972 frame.set_rotation(kVideoRotation_270);
6973 video_source_.IncomingCapturedFrame(frame);
6974 WaitForEncodedFrame(timestamp_ms);
6975 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6976
6977 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006978 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02006979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006980 /*target_bitrate=*/new_stable_rate,
6981 /*stable_target_bitrate=*/new_stable_rate,
6982 /*link_allocation=*/rate,
6983 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006984 /*rtt_ms=*/0,
6985 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006986 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6987 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
6988 video_stream_encoder_->Stop();
6989}
6990
6991TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006992 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006993 const int kFrameWidth = 320;
6994 const int kFrameHeight = 180;
6995
6996 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006997 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006999 /*target_bitrate=*/rate,
7000 /*stable_target_bitrate=*/rate,
7001 /*link_allocation=*/rate,
7002 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007003 /*rtt_ms=*/0,
7004 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007005
7006 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007007 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007008 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7009 frame.set_rotation(kVideoRotation_270);
7010 video_source_.IncomingCapturedFrame(frame);
7011 WaitForEncodedFrame(timestamp_ms);
7012 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7013
7014 // Set a higher target rate without changing the link_allocation. Should not
7015 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007016 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007018 /*target_bitrate=*/rate,
7019 /*stable_target_bitrate=*/new_stable_rate,
7020 /*link_allocation=*/rate,
7021 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007022 /*rtt_ms=*/0,
7023 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007024 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7025 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7026 video_stream_encoder_->Stop();
7027}
7028
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007029TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7030 test::ScopedFieldTrials field_trials(
7031 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7032 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7033 const int kFramerateFps = 30;
7034 const int kWidth = 1920;
7035 const int kHeight = 1080;
7036 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7037 // Works on screenshare mode.
7038 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7039 // We rely on the automatic resolution adaptation, but we handle framerate
7040 // adaptation manually by mocking the stats proxy.
7041 video_source_.set_adaptation_enabled(true);
7042
7043 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007044 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007045 DataRate::BitsPerSec(kTargetBitrateBps),
7046 DataRate::BitsPerSec(kTargetBitrateBps),
7047 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007048 video_stream_encoder_->SetSource(&video_source_,
7049 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007050 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007051
7052 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7053 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7054
7055 // Pass enough frames with the full update to trigger animation detection.
7056 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007057 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007058 frame.set_ntp_time_ms(timestamp_ms);
7059 frame.set_timestamp_us(timestamp_ms * 1000);
7060 video_source_.IncomingCapturedFrame(frame);
7061 WaitForEncodedFrame(timestamp_ms);
7062 }
7063
7064 // Resolution should be limited.
7065 rtc::VideoSinkWants expected;
7066 expected.max_framerate_fps = kFramerateFps;
7067 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007068 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007069
7070 // Pass one frame with no known update.
7071 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007072 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007073 frame.set_ntp_time_ms(timestamp_ms);
7074 frame.set_timestamp_us(timestamp_ms * 1000);
7075 frame.clear_update_rect();
7076
7077 video_source_.IncomingCapturedFrame(frame);
7078 WaitForEncodedFrame(timestamp_ms);
7079
7080 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007081 EXPECT_THAT(video_source_.sink_wants(),
7082 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007083
7084 video_stream_encoder_->Stop();
7085}
7086
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007087TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7088 const int kWidth = 720; // 540p adapted down.
7089 const int kHeight = 405;
7090 const int kNumFrames = 3;
7091 // Works on screenshare mode.
7092 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7093 /*num_spatial_layers=*/2, /*screenshare=*/true);
7094
7095 video_source_.set_adaptation_enabled(true);
7096
7097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7098 DataRate::BitsPerSec(kTargetBitrateBps),
7099 DataRate::BitsPerSec(kTargetBitrateBps),
7100 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7101
7102 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7103
7104 // Pass enough frames with the full update to trigger animation detection.
7105 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007106 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007107 frame.set_ntp_time_ms(timestamp_ms);
7108 frame.set_timestamp_us(timestamp_ms * 1000);
7109 video_source_.IncomingCapturedFrame(frame);
7110 WaitForEncodedFrame(timestamp_ms);
7111 }
7112
7113 video_stream_encoder_->Stop();
7114}
7115
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007116TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7117 const float downscale_factors[] = {4.0, 2.0, 1.0};
7118 const int number_layers =
7119 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7120 VideoEncoderConfig config;
7121 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7122 for (int i = 0; i < number_layers; ++i) {
7123 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7124 config.simulcast_layers[i].active = true;
7125 }
7126 config.video_stream_factory =
7127 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7128 "VP8", /*max qp*/ 56, /*screencast*/ false,
7129 /*screenshare enabled*/ false);
7130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7131 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7132 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7133 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7134
7135 // First initialization.
7136 // Encoder should be initialized. Next frame should be key frame.
7137 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7138 sink_.SetNumExpectedLayers(number_layers);
7139 int64_t timestamp_ms = kFrameIntervalMs;
7140 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7141 WaitForEncodedFrame(timestamp_ms);
7142 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7143 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7144 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7145 VideoFrameType::kVideoFrameKey,
7146 VideoFrameType::kVideoFrameKey}));
7147
7148 // Disable top layer.
7149 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7150 config.simulcast_layers[number_layers - 1].active = false;
7151 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7152 sink_.SetNumExpectedLayers(number_layers - 1);
7153 timestamp_ms += kFrameIntervalMs;
7154 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7155 WaitForEncodedFrame(timestamp_ms);
7156 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7157 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7158 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7159 VideoFrameType::kVideoFrameDelta,
7160 VideoFrameType::kVideoFrameDelta}));
7161
7162 // Re-enable top layer.
7163 // Encoder should be re-initialized. Next frame should be key frame.
7164 config.simulcast_layers[number_layers - 1].active = true;
7165 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7166 sink_.SetNumExpectedLayers(number_layers);
7167 timestamp_ms += kFrameIntervalMs;
7168 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7169 WaitForEncodedFrame(timestamp_ms);
7170 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7171 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7172 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7173 VideoFrameType::kVideoFrameKey,
7174 VideoFrameType::kVideoFrameKey}));
7175
7176 // Top layer max rate change.
7177 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7178 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7179 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7180 sink_.SetNumExpectedLayers(number_layers);
7181 timestamp_ms += kFrameIntervalMs;
7182 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7183 WaitForEncodedFrame(timestamp_ms);
7184 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7185 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7186 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7187 VideoFrameType::kVideoFrameDelta,
7188 VideoFrameType::kVideoFrameDelta}));
7189
7190 // Top layer resolution change.
7191 // Encoder should be re-initialized. Next frame should be key frame.
7192 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7193 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7194 sink_.SetNumExpectedLayers(number_layers);
7195 timestamp_ms += kFrameIntervalMs;
7196 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7197 WaitForEncodedFrame(timestamp_ms);
7198 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7199 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7200 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7201 VideoFrameType::kVideoFrameKey,
7202 VideoFrameType::kVideoFrameKey}));
7203 video_stream_encoder_->Stop();
7204}
7205
perkj26091b12016-09-01 01:17:40 -07007206} // namespace webrtc