blob: 85be6951e5e39f6bc2a3eeff3ee4e267ea352bb3 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020040#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020043#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010044#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020045#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020060using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020062using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020063using ::testing::Ge;
64using ::testing::Gt;
65using ::testing::Le;
66using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010067using ::testing::Matcher;
68using ::testing::NiceMock;
69using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010070using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020071using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080072
perkj803d97f2016-11-01 11:45:46 -070073namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010075const int kQpLow = 1;
76const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020077const int kMinFramerateFps = 2;
78const int kMinBalancedFramerateFps = 7;
79const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080080const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020082const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010083const uint32_t kSimulcastTargetBitrateBps = 3150000;
84const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080085const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070086const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020088const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020089const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
91const VideoEncoder::ResolutionBitrateLimits
92 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020094uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
95 0x00, 0x00, 0x03, 0x03, 0xF4,
96 0x05, 0x03, 0xC7, 0xE0, 0x1B,
97 0x41, 0x10, 0x8D, 0x00};
98
perkj803d97f2016-11-01 11:45:46 -070099class TestBuffer : public webrtc::I420Buffer {
100 public:
101 TestBuffer(rtc::Event* event, int width, int height)
102 : I420Buffer(width, height), event_(event) {}
103
104 private:
105 friend class rtc::RefCountedObject<TestBuffer>;
106 ~TestBuffer() override {
107 if (event_)
108 event_->Set();
109 }
110 rtc::Event* const event_;
111};
112
Noah Richards51db4212019-06-12 06:59:12 -0700113// A fake native buffer that can't be converted to I420.
114class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
115 public:
116 FakeNativeBuffer(rtc::Event* event, int width, int height)
117 : event_(event), width_(width), height_(height) {}
118 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
119 int width() const override { return width_; }
120 int height() const override { return height_; }
121 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
122 return nullptr;
123 }
124
125 private:
126 friend class rtc::RefCountedObject<FakeNativeBuffer>;
127 ~FakeNativeBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132 const int width_;
133 const int height_;
134};
135
Evan Shrubsole895556e2020-10-05 09:15:13 +0200136// A fake native buffer that is backed by an NV12 buffer.
137class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
140 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
141
142 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
143 int width() const override { return nv12_buffer_->width(); }
144 int height() const override { return nv12_buffer_->height(); }
145 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
146 return nv12_buffer_->ToI420();
147 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200148 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
149 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
150 if (absl::c_find(types, Type::kNV12) != types.end()) {
151 return nv12_buffer_;
152 }
153 return nullptr;
154 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200155 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
156
157 private:
158 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
159 ~FakeNV12NativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
164 rtc::Event* const event_;
165};
166
Niels Möller7dc26b72017-12-06 10:27:48 +0100167class CpuOveruseDetectorProxy : public OveruseFrameDetector {
168 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200169 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
170 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200171 last_target_framerate_fps_(-1),
172 framerate_updated_event_(true /* manual_reset */,
173 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100174 virtual ~CpuOveruseDetectorProxy() {}
175
176 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200177 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100178 last_target_framerate_fps_ = framerate_fps;
179 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200180 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100181 }
182
183 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200184 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100185 return last_target_framerate_fps_;
186 }
187
Niels Möller4db138e2018-04-19 09:04:13 +0200188 CpuOveruseOptions GetOptions() { return options_; }
189
Henrik Boström381d1092020-05-12 18:49:07 +0200190 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
191
Niels Möller7dc26b72017-12-06 10:27:48 +0100192 private:
Markus Handella3765182020-07-08 13:13:32 +0200193 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100194 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200195 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100196};
197
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200198class FakeVideoSourceRestrictionsListener
199 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200200 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200201 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200202 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200203 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200204 RTC_DCHECK(was_restrictions_updated_);
205 }
206
207 rtc::Event* restrictions_updated_event() {
208 return &restrictions_updated_event_;
209 }
210
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200211 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200212 void OnVideoSourceRestrictionsUpdated(
213 VideoSourceRestrictions restrictions,
214 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200215 rtc::scoped_refptr<Resource> reason,
216 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200217 was_restrictions_updated_ = true;
218 restrictions_updated_event_.Set();
219 }
220
221 private:
222 bool was_restrictions_updated_;
223 rtc::Event restrictions_updated_event_;
224};
225
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200226auto WantsFps(Matcher<int> fps_matcher) {
227 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
228 fps_matcher);
229}
230
231auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
232 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
233 AllOf(max_pixel_matcher, Gt(0)));
234}
235
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200236auto ResolutionMax() {
237 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
240 Eq(absl::nullopt)));
241}
242
243auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245}
246
247auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200249}
250
251auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200253}
254
255auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200256 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200257}
258
259auto FpsMaxResolutionMax() {
260 return AllOf(FpsMax(), ResolutionMax());
261}
262
263auto UnlimitedSinkWants() {
264 return AllOf(FpsUnlimited(), ResolutionMax());
265}
266
267auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
268 Matcher<int> fps_range_matcher;
269
270 if (last_frame_pixels <= 320 * 240) {
271 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200272 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200273 fps_range_matcher = AllOf(Ge(10), Le(15));
274 } else if (last_frame_pixels <= 640 * 480) {
275 fps_range_matcher = Ge(15);
276 } else {
277 fps_range_matcher = Eq(kDefaultFramerate);
278 }
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_range_matcher);
281}
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
284 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
285 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
286}
287
288auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
289 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
290}
291
292auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
293 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
294}
295
296auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
299}
300
301auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
304}
305
306auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
307 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
308 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
309}
310
311auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
312 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
313 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
314}
315
mflodmancc3d4422017-08-03 08:27:51 -0700316class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700317 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200318 VideoStreamEncoderUnderTest(TimeController* time_controller,
319 TaskQueueFactory* task_queue_factory,
320 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100321 const VideoStreamEncoderSettings& settings,
322 VideoStreamEncoder::BitrateAllocationCallbackType
323 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200324 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100325 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200326 stats_proxy,
327 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200328 std::unique_ptr<OveruseFrameDetector>(
329 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100330 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100331 task_queue_factory,
332 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200333 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200334 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200335 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200336 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200337 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200338 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200339 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200340 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100341 }
perkj803d97f2016-11-01 11:45:46 -0700342
Henrik Boström381d1092020-05-12 18:49:07 +0200343 void SetSourceAndWaitForRestrictionsUpdated(
344 rtc::VideoSourceInterface<VideoFrame>* source,
345 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200346 FakeVideoSourceRestrictionsListener listener;
347 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200348 SetSource(source, degradation_preference);
349 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200350 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200351 }
352
353 void SetSourceAndWaitForFramerateUpdated(
354 rtc::VideoSourceInterface<VideoFrame>* source,
355 const DegradationPreference& degradation_preference) {
356 overuse_detector_proxy_->framerate_updated_event()->Reset();
357 SetSource(source, degradation_preference);
358 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
359 }
360
361 void OnBitrateUpdatedAndWaitForManagedResources(
362 DataRate target_bitrate,
363 DataRate stable_target_bitrate,
364 DataRate link_allocation,
365 uint8_t fraction_lost,
366 int64_t round_trip_time_ms,
367 double cwnd_reduce_ratio) {
368 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
369 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
370 // Bitrate is updated on the encoder queue.
371 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200372 }
373
kthelgason2fc52542017-03-03 00:24:41 -0800374 // This is used as a synchronisation mechanism, to make sure that the
375 // encoder queue is not blocked before we start sending it frames.
376 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100377 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200378 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800379 ASSERT_TRUE(event.Wait(5000));
380 }
381
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200383 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200384 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200385 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200386 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200387 event.Set();
388 });
389 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200390 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200391 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200392
Henrik Boström91aa7322020-04-28 12:24:33 +0200393 void TriggerCpuUnderuse() {
394 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200395 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200396 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200397 event.Set();
398 });
399 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200400 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200401 }
kthelgason876222f2016-11-29 01:44:11 -0800402
Henrik Boström91aa7322020-04-28 12:24:33 +0200403 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200404 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200405 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200406 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200407 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 event.Set();
409 });
410 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200411 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200412 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200413 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200415 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200416 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 event.Set();
418 });
419 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200421 }
422
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200423 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100424 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200425 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
426 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200427 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700428};
429
Noah Richards51db4212019-06-12 06:59:12 -0700430// Simulates simulcast behavior and makes highest stream resolutions divisible
431// by 4.
432class CroppingVideoStreamFactory
433 : public VideoEncoderConfig::VideoStreamFactoryInterface {
434 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200435 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700436
437 private:
438 std::vector<VideoStream> CreateEncoderStreams(
439 int width,
440 int height,
441 const VideoEncoderConfig& encoder_config) override {
442 std::vector<VideoStream> streams = test::CreateVideoStreams(
443 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700444 return streams;
445 }
Noah Richards51db4212019-06-12 06:59:12 -0700446};
447
sprangb1ca0732017-02-01 08:38:12 -0800448class AdaptingFrameForwarder : public test::FrameForwarder {
449 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 explicit AdaptingFrameForwarder(TimeController* time_controller)
451 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700452 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800453
454 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200455 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800456 adaptation_enabled_ = enabled;
457 }
458
asaperssonfab67072017-04-04 05:51:49 -0700459 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200460 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800461 return adaptation_enabled_;
462 }
463
asapersson09f05612017-05-15 23:40:18 -0700464 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200465 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700466 return last_wants_;
467 }
468
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200469 absl::optional<int> last_sent_width() const { return last_width_; }
470 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800471
sprangb1ca0732017-02-01 08:38:12 -0800472 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200473 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
474 time_controller_->AdvanceTime(TimeDelta::Millis(0));
475
sprangb1ca0732017-02-01 08:38:12 -0800476 int cropped_width = 0;
477 int cropped_height = 0;
478 int out_width = 0;
479 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700480 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200481 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
482 << "w=" << video_frame.width()
483 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700484 if (adapter_.AdaptFrameResolution(
485 video_frame.width(), video_frame.height(),
486 video_frame.timestamp_us() * 1000, &cropped_width,
487 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100488 VideoFrame adapted_frame =
489 VideoFrame::Builder()
490 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
491 nullptr, out_width, out_height))
492 .set_timestamp_rtp(99)
493 .set_timestamp_ms(99)
494 .set_rotation(kVideoRotation_0)
495 .build();
sprangc5d62e22017-04-02 23:53:04 -0700496 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100497 if (video_frame.has_update_rect()) {
498 adapted_frame.set_update_rect(
499 video_frame.update_rect().ScaleWithFrame(
500 video_frame.width(), video_frame.height(), 0, 0,
501 video_frame.width(), video_frame.height(), out_width,
502 out_height));
503 }
sprangc5d62e22017-04-02 23:53:04 -0700504 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800505 last_width_.emplace(adapted_frame.width());
506 last_height_.emplace(adapted_frame.height());
507 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200508 last_width_ = absl::nullopt;
509 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700510 }
sprangb1ca0732017-02-01 08:38:12 -0800511 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800513 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800514 last_width_.emplace(video_frame.width());
515 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800516 }
517 }
518
519 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
520 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200522 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100523 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200524 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800525 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200526
527 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800528 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200529 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
530 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200531 absl::optional<int> last_width_;
532 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800533};
sprangc5d62e22017-04-02 23:53:04 -0700534
Niels Möller213618e2018-07-24 09:29:58 +0200535// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700536class MockableSendStatisticsProxy : public SendStatisticsProxy {
537 public:
538 MockableSendStatisticsProxy(Clock* clock,
539 const VideoSendStream::Config& config,
540 VideoEncoderConfig::ContentType content_type)
541 : SendStatisticsProxy(clock, config, content_type) {}
542
543 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200544 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700545 if (mock_stats_)
546 return *mock_stats_;
547 return SendStatisticsProxy::GetStats();
548 }
549
Niels Möller213618e2018-07-24 09:29:58 +0200550 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200551 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200552 if (mock_stats_)
553 return mock_stats_->input_frame_rate;
554 return SendStatisticsProxy::GetInputFrameRate();
555 }
sprangc5d62e22017-04-02 23:53:04 -0700556 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200557 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700558 mock_stats_.emplace(stats);
559 }
560
561 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200562 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700563 mock_stats_.reset();
564 }
565
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200566 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
567 on_frame_dropped_ = std::move(callback);
568 }
569
sprangc5d62e22017-04-02 23:53:04 -0700570 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200571 void OnFrameDropped(DropReason reason) override {
572 SendStatisticsProxy::OnFrameDropped(reason);
573 if (on_frame_dropped_)
574 on_frame_dropped_(reason);
575 }
576
Markus Handella3765182020-07-08 13:13:32 +0200577 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200578 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200579 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700580};
581
philipel9b058032020-02-10 11:30:00 +0100582class MockEncoderSelector
583 : public VideoEncoderFactory::EncoderSelectorInterface {
584 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200585 MOCK_METHOD(void,
586 OnCurrentEncoder,
587 (const SdpVideoFormat& format),
588 (override));
589 MOCK_METHOD(absl::optional<SdpVideoFormat>,
590 OnAvailableBitrate,
591 (const DataRate& rate),
592 (override));
593 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100594};
595
perkj803d97f2016-11-01 11:45:46 -0700596} // namespace
597
mflodmancc3d4422017-08-03 08:27:51 -0700598class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700599 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200600 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700601
mflodmancc3d4422017-08-03 08:27:51 -0700602 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700603 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700604 codec_width_(320),
605 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200606 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200608 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700609 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700611 video_send_config_,
612 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200613 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700614
615 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700616 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700617 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200618 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800619 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200620 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200621 video_send_config_.rtp.payload_name = "FAKE";
622 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700623
Per512ecb32016-09-23 15:52:06 +0200624 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200625 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200626 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
627 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
628 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100629 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700630
Niels Möllerf1338562018-04-26 09:51:47 +0200631 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800632 }
633
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100634 void ConfigureEncoder(
635 VideoEncoderConfig video_encoder_config,
636 VideoStreamEncoder::BitrateAllocationCallbackType
637 allocation_callback_type =
638 VideoStreamEncoder::BitrateAllocationCallbackType::
639 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700640 if (video_stream_encoder_)
641 video_stream_encoder_->Stop();
642 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200643 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100644 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700645 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700647 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700648 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
649 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200650 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800652 }
653
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100654 void ResetEncoder(const std::string& payload_name,
655 size_t num_streams,
656 size_t num_temporal_layers,
657 unsigned char num_spatial_layers,
658 bool screenshare,
659 VideoStreamEncoder::BitrateAllocationCallbackType
660 allocation_callback_type =
661 VideoStreamEncoder::BitrateAllocationCallbackType::
662 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200663 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800664
665 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200666 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
667 num_streams, &video_encoder_config);
668 for (auto& layer : video_encoder_config.simulcast_layers) {
669 layer.num_temporal_layers = num_temporal_layers;
670 layer.max_framerate = kDefaultFramerate;
671 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100672 video_encoder_config.max_bitrate_bps =
673 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700674 video_encoder_config.content_type =
675 screenshare ? VideoEncoderConfig::ContentType::kScreen
676 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700677 if (payload_name == "VP9") {
678 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
679 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200680 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700681 video_encoder_config.encoder_specific_settings =
682 new rtc::RefCountedObject<
683 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
684 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100685 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700686 }
687
sprang57c2fff2017-01-16 06:24:02 -0800688 VideoFrame CreateFrame(int64_t ntp_time_ms,
689 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100690 VideoFrame frame =
691 VideoFrame::Builder()
692 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
693 destruction_event, codec_width_, codec_height_))
694 .set_timestamp_rtp(99)
695 .set_timestamp_ms(99)
696 .set_rotation(kVideoRotation_0)
697 .build();
sprang57c2fff2017-01-16 06:24:02 -0800698 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700699 return frame;
700 }
701
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100702 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
703 rtc::Event* destruction_event,
704 int offset_x) const {
705 VideoFrame frame =
706 VideoFrame::Builder()
707 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
708 destruction_event, codec_width_, codec_height_))
709 .set_timestamp_rtp(99)
710 .set_timestamp_ms(99)
711 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100712 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100713 .build();
714 frame.set_ntp_time_ms(ntp_time_ms);
715 return frame;
716 }
717
sprang57c2fff2017-01-16 06:24:02 -0800718 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100719 VideoFrame frame =
720 VideoFrame::Builder()
721 .set_video_frame_buffer(
722 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
723 .set_timestamp_rtp(99)
724 .set_timestamp_ms(99)
725 .set_rotation(kVideoRotation_0)
726 .build();
sprang57c2fff2017-01-16 06:24:02 -0800727 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700728 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700729 return frame;
730 }
731
Evan Shrubsole895556e2020-10-05 09:15:13 +0200732 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
733 VideoFrame frame =
734 VideoFrame::Builder()
735 .set_video_frame_buffer(NV12Buffer::Create(width, height))
736 .set_timestamp_rtp(99)
737 .set_timestamp_ms(99)
738 .set_rotation(kVideoRotation_0)
739 .build();
740 frame.set_ntp_time_ms(ntp_time_ms);
741 frame.set_timestamp_us(ntp_time_ms * 1000);
742 return frame;
743 }
744
Noah Richards51db4212019-06-12 06:59:12 -0700745 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int width,
748 int height) const {
749 VideoFrame frame =
750 VideoFrame::Builder()
751 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
752 destruction_event, width, height))
753 .set_timestamp_rtp(99)
754 .set_timestamp_ms(99)
755 .set_rotation(kVideoRotation_0)
756 .build();
757 frame.set_ntp_time_ms(ntp_time_ms);
758 return frame;
759 }
760
Evan Shrubsole895556e2020-10-05 09:15:13 +0200761 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
762 rtc::Event* destruction_event,
763 int width,
764 int height) const {
765 VideoFrame frame = VideoFrame::Builder()
766 .set_video_frame_buffer(
767 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
768 destruction_event, width, height))
769 .set_timestamp_rtp(99)
770 .set_timestamp_ms(99)
771 .set_rotation(kVideoRotation_0)
772 .build();
773 frame.set_ntp_time_ms(ntp_time_ms);
774 return frame;
775 }
776
Noah Richards51db4212019-06-12 06:59:12 -0700777 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
778 rtc::Event* destruction_event) const {
779 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
780 codec_height_);
781 }
782
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100783 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100785 DataRate::BitsPerSec(kTargetBitrateBps),
786 DataRate::BitsPerSec(kTargetBitrateBps),
787 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100788
789 video_source_.IncomingCapturedFrame(
790 CreateFrame(1, codec_width_, codec_height_));
791 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200792 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100793 }
794
sprang4847ae62017-06-27 07:06:52 -0700795 void WaitForEncodedFrame(int64_t expected_ntp_time) {
796 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200797 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700798 }
799
800 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
801 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200802 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700803 return ok;
804 }
805
806 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
807 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200808 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700809 }
810
811 void ExpectDroppedFrame() {
812 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200813 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 }
815
816 bool WaitForFrame(int64_t timeout_ms) {
817 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200818 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700819 return ok;
820 }
821
perkj26091b12016-09-01 01:17:40 -0700822 class TestEncoder : public test::FakeEncoder {
823 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 explicit TestEncoder(TimeController* time_controller)
825 : FakeEncoder(time_controller->GetClock()),
826 time_controller_(time_controller) {
827 RTC_DCHECK(time_controller_);
828 }
perkj26091b12016-09-01 01:17:40 -0700829
asaperssonfab67072017-04-04 05:51:49 -0700830 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200831 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700832 return config_;
833 }
834
835 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200836 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700837 block_next_encode_ = true;
838 }
839
Erik Språngaed30702018-11-05 12:57:17 +0100840 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200841 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200842 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100843 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100845 info.scaling_settings = VideoEncoder::ScalingSettings(
846 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100847 }
848 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 for (int i = 0; i < kMaxSpatialLayers; ++i) {
850 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100851 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100852 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100853 for (int tid = 0; tid < num_layers; ++tid)
854 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100855 }
856 }
Erik Språngaed30702018-11-05 12:57:17 +0100857 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200858
859 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100860 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200861 info.apply_alignment_to_all_simulcast_layers =
862 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200863 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100864 return info;
kthelgason876222f2016-11-29 01:44:11 -0800865 }
866
Erik Språngb7cb7b52019-02-26 15:52:33 +0100867 int32_t RegisterEncodeCompleteCallback(
868 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200869 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100870 encoded_image_callback_ = callback;
871 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
872 }
873
perkjfa10b552016-10-02 23:45:26 -0700874 void ContinueEncode() { continue_encode_event_.Set(); }
875
876 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
877 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200878 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700879 EXPECT_EQ(timestamp_, timestamp);
880 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
881 }
882
kthelgason2fc52542017-03-03 00:24:41 -0800883 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200884 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800885 quality_scaling_ = b;
886 }
kthelgasonad9010c2017-02-14 00:46:51 -0800887
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100888 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200889 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100890 requested_resolution_alignment_ = requested_resolution_alignment;
891 }
892
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200893 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
894 MutexLock lock(&local_mutex_);
895 apply_alignment_to_all_simulcast_layers_ = b;
896 }
897
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100898 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200899 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100900 is_hardware_accelerated_ = is_hardware_accelerated;
901 }
902
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100903 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
904 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100906 temporal_layers_supported_[spatial_idx] = supported;
907 }
908
Sergey Silkin6456e352019-07-08 17:56:40 +0200909 void SetResolutionBitrateLimits(
910 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200911 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200912 resolution_bitrate_limits_ = thresholds;
913 }
914
sprangfe627f32017-03-29 08:24:59 -0700915 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700917 force_init_encode_failed_ = force_failure;
918 }
919
Niels Möller6bb5ab92019-01-11 11:11:10 +0100920 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100922 rate_factor_ = rate_factor;
923 }
924
Erik Språngd7329ca2019-02-21 21:19:53 +0100925 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100927 return last_framerate_;
928 }
929
Erik Språngd7329ca2019-02-21 21:19:53 +0100930 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200931 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100932 return last_update_rect_;
933 }
934
Niels Möller87e2d782019-03-07 10:18:23 +0100935 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200936 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100937 return last_frame_types_;
938 }
939
940 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100941 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100942 keyframe ? VideoFrameType::kVideoFrameKey
943 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100944 {
Markus Handella3765182020-07-08 13:13:32 +0200945 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100946 last_frame_types_ = frame_type;
947 }
Niels Möllerb859b322019-03-07 12:40:01 +0100948 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100949 }
950
Erik Språngb7cb7b52019-02-26 15:52:33 +0100951 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200952 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200953 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100954 }
955
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200956 void SetEncodedImageData(
957 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200958 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200959 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200960 }
961
Erik Språngd7329ca2019-02-21 21:19:53 +0100962 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 expect_null_frame_ = true;
965 }
966
Erik Språng5056af02019-09-02 15:53:11 +0200967 absl::optional<VideoEncoder::RateControlParameters>
968 GetAndResetLastRateControlSettings() {
969 auto settings = last_rate_control_settings_;
970 last_rate_control_settings_.reset();
971 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100972 }
973
Evan Shrubsole895556e2020-10-05 09:15:13 +0200974 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
975 MutexLock lock(&local_mutex_);
976 return last_input_pixel_format_;
977 }
978
Sergey Silkin5ee69672019-07-02 14:18:34 +0200979 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200980 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200981 return num_encoder_initializations_;
982 }
983
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200984 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200985 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200986 return num_set_rates_;
987 }
988
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200989 VideoCodec video_codec() const {
990 MutexLock lock(&local_mutex_);
991 return video_codec_;
992 }
993
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200994 void SetPreferredPixelFormats(
995 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
996 pixel_formats) {
997 MutexLock lock(&local_mutex_);
998 preferred_pixel_formats_ = std::move(pixel_formats);
999 }
1000
perkjfa10b552016-10-02 23:45:26 -07001001 private:
perkj26091b12016-09-01 01:17:40 -07001002 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001003 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001004 bool block_encode;
1005 {
Markus Handella3765182020-07-08 13:13:32 +02001006 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001007 if (expect_null_frame_) {
1008 EXPECT_EQ(input_image.timestamp(), 0u);
1009 EXPECT_EQ(input_image.width(), 1);
1010 last_frame_types_ = *frame_types;
1011 expect_null_frame_ = false;
1012 } else {
1013 EXPECT_GT(input_image.timestamp(), timestamp_);
1014 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1015 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1016 }
perkj26091b12016-09-01 01:17:40 -07001017
1018 timestamp_ = input_image.timestamp();
1019 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001020 last_input_width_ = input_image.width();
1021 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001022 block_encode = block_next_encode_;
1023 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001024 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001025 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001026 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001027 }
Niels Möllerb859b322019-03-07 12:40:01 +01001028 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001029 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001030 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001031
perkj26091b12016-09-01 01:17:40 -07001032 return result;
1033 }
1034
Niels Möller08ae7ce2020-09-23 15:58:12 +02001035 CodecSpecificInfo EncodeHook(
1036 EncodedImage& encoded_image,
1037 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001038 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001039 {
1040 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001041 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001042 }
1043 MutexLock lock(&local_mutex_);
1044 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001045 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001046 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001047 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001048 }
1049
sprangfe627f32017-03-29 08:24:59 -07001050 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001051 const Settings& settings) override {
1052 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001053
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001055 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001056
1057 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001058 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001059
Erik Språng82fad3d2018-03-21 09:57:23 +01001060 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001061 // Simulate setting up temporal layers, in order to validate the life
1062 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001063 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001064 frame_buffer_controller_ =
1065 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001066 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001067 if (force_init_encode_failed_) {
1068 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001069 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001070 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001071
Erik Språngb7cb7b52019-02-26 15:52:33 +01001072 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001073 return res;
1074 }
1075
Erik Språngb7cb7b52019-02-26 15:52:33 +01001076 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001077 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1079 initialized_ = EncoderState::kUninitialized;
1080 return FakeEncoder::Release();
1081 }
1082
Erik Språng16cb8f52019-04-12 13:59:09 +02001083 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001085 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001086 VideoBitrateAllocation adjusted_rate_allocation;
1087 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1088 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001089 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001090 adjusted_rate_allocation.SetBitrate(
1091 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001092 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001093 rate_factor_));
1094 }
1095 }
1096 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001097 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001098 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001099 RateControlParameters adjusted_paramters = parameters;
1100 adjusted_paramters.bitrate = adjusted_rate_allocation;
1101 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001102 }
1103
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001104 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001105 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001106 enum class EncoderState {
1107 kUninitialized,
1108 kInitializationFailed,
1109 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001110 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1111 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001112 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001113 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1114 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1115 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1116 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1117 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1118 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001119 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1120 false;
Markus Handella3765182020-07-08 13:13:32 +02001121 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001122 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1123 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001124 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001125 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001126 absl::optional<bool>
1127 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001128 local_mutex_);
1129 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1130 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1131 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001132 absl::optional<VideoEncoder::RateControlParameters>
1133 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001134 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1135 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001136 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001137 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001138 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1139 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001140 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001141 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001142 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001143 RTC_GUARDED_BY(local_mutex_);
1144 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001145 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001146 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1147 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001148 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1149 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001150 };
1151
mflodmancc3d4422017-08-03 08:27:51 -07001152 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001153 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001154 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1155 : time_controller_(time_controller), test_encoder_(test_encoder) {
1156 RTC_DCHECK(time_controller_);
1157 }
perkj26091b12016-09-01 01:17:40 -07001158
perkj26091b12016-09-01 01:17:40 -07001159 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001160 EXPECT_TRUE(
1161 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1162 }
1163
1164 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1165 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001166 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001167 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001168 return false;
perkj26091b12016-09-01 01:17:40 -07001169 {
Markus Handella3765182020-07-08 13:13:32 +02001170 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001171 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001172 }
1173 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001174 return true;
perkj26091b12016-09-01 01:17:40 -07001175 }
1176
sprangb1ca0732017-02-01 08:38:12 -08001177 void WaitForEncodedFrame(uint32_t expected_width,
1178 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001179 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001180 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001181 }
1182
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001183 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001184 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001185 uint32_t width = 0;
1186 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001187 {
Markus Handella3765182020-07-08 13:13:32 +02001188 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001189 width = last_width_;
1190 height = last_height_;
1191 }
1192 EXPECT_EQ(expected_height, height);
1193 EXPECT_EQ(expected_width, width);
1194 }
1195
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001196 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1197 VideoRotation rotation;
1198 {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001200 rotation = last_rotation_;
1201 }
1202 EXPECT_EQ(expected_rotation, rotation);
1203 }
1204
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001205 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001206
sprangc5d62e22017-04-02 23:53:04 -07001207 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001208 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1209 bool ret = encoded_frame_event_.Wait(timeout_ms);
1210 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1211 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001212 }
1213
perkj26091b12016-09-01 01:17:40 -07001214 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001215 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001216 expect_frames_ = false;
1217 }
1218
asaperssonfab67072017-04-04 05:51:49 -07001219 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001220 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001221 return number_of_reconfigurations_;
1222 }
1223
asaperssonfab67072017-04-04 05:51:49 -07001224 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001225 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001226 return min_transmit_bitrate_bps_;
1227 }
1228
Erik Språngd7329ca2019-02-21 21:19:53 +01001229 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001230 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001231 num_expected_layers_ = num_layers;
1232 }
1233
Erik Språngb7cb7b52019-02-26 15:52:33 +01001234 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001235 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001236 return last_capture_time_ms_;
1237 }
1238
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001239 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001240 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001241 return std::move(last_encoded_image_data_);
1242 }
1243
Per Kjellanderdcef6412020-10-07 15:09:05 +02001244 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1245 MutexLock lock(&mutex_);
1246 return last_bitrate_allocation_;
1247 }
1248
1249 int number_of_bitrate_allocations() const {
1250 MutexLock lock(&mutex_);
1251 return number_of_bitrate_allocations_;
1252 }
1253
Per Kjellandera9434842020-10-15 17:53:22 +02001254 VideoLayersAllocation GetLastVideoLayersAllocation() {
1255 MutexLock lock(&mutex_);
1256 return last_layers_allocation_;
1257 }
1258
1259 int number_of_layers_allocations() const {
1260 MutexLock lock(&mutex_);
1261 return number_of_layers_allocations_;
1262 }
1263
perkj26091b12016-09-01 01:17:40 -07001264 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001265 Result OnEncodedImage(
1266 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001267 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001269 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001270 last_encoded_image_data_ = std::vector<uint8_t>(
1271 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001272 uint32_t timestamp = encoded_image.Timestamp();
1273 if (last_timestamp_ != timestamp) {
1274 num_received_layers_ = 1;
1275 } else {
1276 ++num_received_layers_;
1277 }
1278 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001279 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001280 last_width_ = encoded_image._encodedWidth;
1281 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001282 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001283 if (num_received_layers_ == num_expected_layers_) {
1284 encoded_frame_event_.Set();
1285 }
sprangb1ca0732017-02-01 08:38:12 -08001286 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001287 }
1288
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001289 void OnEncoderConfigurationChanged(
1290 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001291 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001292 VideoEncoderConfig::ContentType content_type,
1293 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001294 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001295 ++number_of_reconfigurations_;
1296 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1297 }
1298
Per Kjellanderdcef6412020-10-07 15:09:05 +02001299 void OnBitrateAllocationUpdated(
1300 const VideoBitrateAllocation& allocation) override {
1301 MutexLock lock(&mutex_);
1302 ++number_of_bitrate_allocations_;
1303 last_bitrate_allocation_ = allocation;
1304 }
1305
Per Kjellandera9434842020-10-15 17:53:22 +02001306 void OnVideoLayersAllocationUpdated(
1307 VideoLayersAllocation allocation) override {
1308 MutexLock lock(&mutex_);
1309 ++number_of_layers_allocations_;
1310 last_layers_allocation_ = allocation;
1311 rtc::StringBuilder log;
1312 for (const auto& layer : allocation.active_spatial_layers) {
1313 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1314 << "[";
1315 for (const auto target_bitrate :
1316 layer.target_bitrate_per_temporal_layer) {
1317 log << target_bitrate.kbps() << ",";
1318 }
1319 log << "]";
1320 }
1321 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1322 }
1323
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001324 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001325 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001326 TestEncoder* test_encoder_;
1327 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001328 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001329 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001330 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001331 uint32_t last_height_ = 0;
1332 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001333 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001334 size_t num_expected_layers_ = 1;
1335 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001336 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001337 int number_of_reconfigurations_ = 0;
1338 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001339 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1340 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001341 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1342 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001343 };
1344
Sergey Silkin5ee69672019-07-02 14:18:34 +02001345 class VideoBitrateAllocatorProxyFactory
1346 : public VideoBitrateAllocatorFactory {
1347 public:
1348 VideoBitrateAllocatorProxyFactory()
1349 : bitrate_allocator_factory_(
1350 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1351
1352 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1353 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001354 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001355 codec_config_ = codec;
1356 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1357 }
1358
1359 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001360 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001361 return codec_config_;
1362 }
1363
1364 private:
1365 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1366
Markus Handella3765182020-07-08 13:13:32 +02001367 mutable Mutex mutex_;
1368 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001369 };
1370
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001371 Clock* clock() { return time_controller_.GetClock(); }
1372 void AdvanceTime(TimeDelta duration) {
1373 time_controller_.AdvanceTime(duration);
1374 }
1375
1376 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1377
1378 protected:
1379 virtual TaskQueueFactory* GetTaskQueueFactory() {
1380 return time_controller_.GetTaskQueueFactory();
1381 }
1382
1383 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001384 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001385 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001386 int codec_width_;
1387 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001388 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001389 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001390 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001391 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001392 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001393 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001394 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001395 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001396};
1397
mflodmancc3d4422017-08-03 08:27:51 -07001398TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001400 DataRate::BitsPerSec(kTargetBitrateBps),
1401 DataRate::BitsPerSec(kTargetBitrateBps),
1402 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001403 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001404 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001405 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001406 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001407 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001408}
1409
mflodmancc3d4422017-08-03 08:27:51 -07001410TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001411 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001412 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001413 // The encoder will cache up to one frame for a short duration. Adding two
1414 // frames means that the first frame will be dropped and the second frame will
1415 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001416 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001417 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001418 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001419 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001420
Henrik Boström381d1092020-05-12 18:49:07 +02001421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001422 DataRate::BitsPerSec(kTargetBitrateBps),
1423 DataRate::BitsPerSec(kTargetBitrateBps),
1424 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001425
Sebastian Janssona3177052018-04-10 13:05:49 +02001426 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001427 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001428 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1429
1430 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001431 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001432}
1433
mflodmancc3d4422017-08-03 08:27:51 -07001434TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps),
1438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001439 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001440 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001441
Henrik Boström381d1092020-05-12 18:49:07 +02001442 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1443 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1444 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001445 // The encoder will cache up to one frame for a short duration. Adding two
1446 // frames means that the first frame will be dropped and the second frame will
1447 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001448 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001449 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001450
Henrik Boström381d1092020-05-12 18:49:07 +02001451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001452 DataRate::BitsPerSec(kTargetBitrateBps),
1453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001455 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001456 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1457 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001458 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001459}
1460
mflodmancc3d4422017-08-03 08:27:51 -07001461TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001463 DataRate::BitsPerSec(kTargetBitrateBps),
1464 DataRate::BitsPerSec(kTargetBitrateBps),
1465 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001466 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001467 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001468
1469 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001470 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001471
perkja49cbd32016-09-16 07:53:41 -07001472 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001473 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001474 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001475}
1476
mflodmancc3d4422017-08-03 08:27:51 -07001477TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001479 DataRate::BitsPerSec(kTargetBitrateBps),
1480 DataRate::BitsPerSec(kTargetBitrateBps),
1481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001482
perkja49cbd32016-09-16 07:53:41 -07001483 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001484 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001485
mflodmancc3d4422017-08-03 08:27:51 -07001486 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001487 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001488 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001489 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1490 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001491}
1492
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001493class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1494 public:
1495 VideoStreamEncoderBlockedTest() {}
1496
1497 TaskQueueFactory* GetTaskQueueFactory() override {
1498 return task_queue_factory_.get();
1499 }
1500
1501 private:
1502 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1503 CreateDefaultTaskQueueFactory();
1504};
1505
1506TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001508 DataRate::BitsPerSec(kTargetBitrateBps),
1509 DataRate::BitsPerSec(kTargetBitrateBps),
1510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001511
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001512 int dropped_count = 0;
1513 stats_proxy_->SetDroppedFrameCallback(
1514 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1515 ++dropped_count;
1516 });
1517
perkj26091b12016-09-01 01:17:40 -07001518 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001519 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001520 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001521 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1522 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001523 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1524 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001525 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001526 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001527
mflodmancc3d4422017-08-03 08:27:51 -07001528 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001529
1530 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001531}
1532
Noah Richards51db4212019-06-12 06:59:12 -07001533TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001538
1539 rtc::Event frame_destroyed_event;
1540 video_source_.IncomingCapturedFrame(
1541 CreateFakeNativeFrame(1, &frame_destroyed_event));
1542 ExpectDroppedFrame();
1543 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1544 video_stream_encoder_->Stop();
1545}
1546
1547TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1548 // Use the cropping factory.
1549 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001550 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001551 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1552 kMaxPayloadLength);
1553 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1554
1555 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001557 DataRate::BitsPerSec(kTargetBitrateBps),
1558 DataRate::BitsPerSec(kTargetBitrateBps),
1559 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001560 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1561 WaitForEncodedFrame(1);
1562 // The encoder will have been configured once.
1563 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1564 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1565 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1566
1567 // Now send in a fake frame that needs to be cropped as the width/height
1568 // aren't divisible by 4 (see CreateEncoderStreams above).
1569 rtc::Event frame_destroyed_event;
1570 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1571 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1572 ExpectDroppedFrame();
1573 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1574 video_stream_encoder_->Stop();
1575}
1576
Evan Shrubsole895556e2020-10-05 09:15:13 +02001577TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1579 DataRate::BitsPerSec(kTargetBitrateBps),
1580 DataRate::BitsPerSec(kTargetBitrateBps),
1581 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1582
1583 video_source_.IncomingCapturedFrame(
1584 CreateNV12Frame(1, codec_width_, codec_height_));
1585 WaitForEncodedFrame(1);
1586 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1587 fake_encoder_.GetLastInputPixelFormat());
1588 video_stream_encoder_->Stop();
1589}
1590
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001591TEST_F(VideoStreamEncoderTest,
1592 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1594 DataRate::BitsPerSec(kTargetBitrateBps),
1595 DataRate::BitsPerSec(kTargetBitrateBps),
1596 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1597
1598 fake_encoder_.SetPreferredPixelFormats({});
1599
1600 rtc::Event frame_destroyed_event;
1601 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1602 1, &frame_destroyed_event, codec_width_, codec_height_));
1603 WaitForEncodedFrame(1);
1604 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1605 fake_encoder_.GetLastInputPixelFormat());
1606 video_stream_encoder_->Stop();
1607}
1608
1609TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1611 DataRate::BitsPerSec(kTargetBitrateBps),
1612 DataRate::BitsPerSec(kTargetBitrateBps),
1613 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1614
1615 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1616
1617 rtc::Event frame_destroyed_event;
1618 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1619 1, &frame_destroyed_event, codec_width_, codec_height_));
1620 WaitForEncodedFrame(1);
1621 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1622 fake_encoder_.GetLastInputPixelFormat());
1623 video_stream_encoder_->Stop();
1624}
1625
1626TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1628 DataRate::BitsPerSec(kTargetBitrateBps),
1629 DataRate::BitsPerSec(kTargetBitrateBps),
1630 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1631
1632 // Fake NV12 native frame does not allow mapping to I444.
1633 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1634
1635 rtc::Event frame_destroyed_event;
1636 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1637 1, &frame_destroyed_event, codec_width_, codec_height_));
1638 WaitForEncodedFrame(1);
1639 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1640 fake_encoder_.GetLastInputPixelFormat());
1641 video_stream_encoder_->Stop();
1642}
1643
Evan Shrubsole895556e2020-10-05 09:15:13 +02001644TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1646 DataRate::BitsPerSec(kTargetBitrateBps),
1647 DataRate::BitsPerSec(kTargetBitrateBps),
1648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1649
1650 rtc::Event frame_destroyed_event;
1651 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1652 1, &frame_destroyed_event, codec_width_, codec_height_));
1653 WaitForEncodedFrame(1);
1654 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1655 fake_encoder_.GetLastInputPixelFormat());
1656 video_stream_encoder_->Stop();
1657}
1658
Ying Wang9b881ab2020-02-07 14:29:32 +01001659TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps),
1663 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001664 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1665 WaitForEncodedFrame(1);
1666
Henrik Boström381d1092020-05-12 18:49:07 +02001667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001668 DataRate::BitsPerSec(kTargetBitrateBps),
1669 DataRate::BitsPerSec(kTargetBitrateBps),
1670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001671 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1672 // frames. Adding two frames means that the first frame will be dropped and
1673 // the second frame will be sent to the encoder.
1674 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1675 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1676 WaitForEncodedFrame(3);
1677 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1678 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1679 WaitForEncodedFrame(5);
1680 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1681 video_stream_encoder_->Stop();
1682}
1683
mflodmancc3d4422017-08-03 08:27:51 -07001684TEST_F(VideoStreamEncoderTest,
1685 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps),
1689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001690 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001691
1692 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001693 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001694 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001695 // The encoder will have been configured once when the first frame is
1696 // received.
1697 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001698
1699 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001700 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001701 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001702 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001703 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001704
1705 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001706 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001707 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001708 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001709 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001710
mflodmancc3d4422017-08-03 08:27:51 -07001711 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001712}
1713
mflodmancc3d4422017-08-03 08:27:51 -07001714TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001716 DataRate::BitsPerSec(kTargetBitrateBps),
1717 DataRate::BitsPerSec(kTargetBitrateBps),
1718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001719
1720 // Capture a frame and wait for it to synchronize with the encoder thread.
1721 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001722 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001723 // The encoder will have been configured once.
1724 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001725 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1726 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1727
1728 codec_width_ *= 2;
1729 codec_height_ *= 2;
1730 // Capture a frame with a higher resolution and wait for it to synchronize
1731 // with the encoder thread.
1732 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001733 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001734 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1735 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001736 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001737
mflodmancc3d4422017-08-03 08:27:51 -07001738 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001739}
1740
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001741TEST_F(VideoStreamEncoderTest,
1742 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001744 DataRate::BitsPerSec(kTargetBitrateBps),
1745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001747
1748 // Capture a frame and wait for it to synchronize with the encoder thread.
1749 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1750 WaitForEncodedFrame(1);
1751
1752 VideoEncoderConfig video_encoder_config;
1753 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1754 // Changing the max payload data length recreates encoder.
1755 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1756 kMaxPayloadLength / 2);
1757
1758 // Capture a frame and wait for it to synchronize with the encoder thread.
1759 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1760 WaitForEncodedFrame(2);
1761 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1762
1763 video_stream_encoder_->Stop();
1764}
1765
Sergey Silkin5ee69672019-07-02 14:18:34 +02001766TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001768 DataRate::BitsPerSec(kTargetBitrateBps),
1769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001771
1772 VideoEncoderConfig video_encoder_config;
1773 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1774 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1775 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1776 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1777 kMaxPayloadLength);
1778
1779 // Capture a frame and wait for it to synchronize with the encoder thread.
1780 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1781 WaitForEncodedFrame(1);
1782 // The encoder will have been configured once when the first frame is
1783 // received.
1784 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1785 EXPECT_EQ(kTargetBitrateBps,
1786 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1787 EXPECT_EQ(kStartBitrateBps,
1788 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1789
Sergey Silkin6456e352019-07-08 17:56:40 +02001790 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1791 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001792 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1793 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1794 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1795 kMaxPayloadLength);
1796
1797 // Capture a frame and wait for it to synchronize with the encoder thread.
1798 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1799 WaitForEncodedFrame(2);
1800 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1801 // Bitrate limits have changed - rate allocator should be reconfigured,
1802 // encoder should not be reconfigured.
1803 EXPECT_EQ(kTargetBitrateBps * 2,
1804 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1805 EXPECT_EQ(kStartBitrateBps * 2,
1806 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1807 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1808
1809 video_stream_encoder_->Stop();
1810}
1811
Sergey Silkin6456e352019-07-08 17:56:40 +02001812TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001813 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001814 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001815 DataRate::BitsPerSec(kTargetBitrateBps),
1816 DataRate::BitsPerSec(kTargetBitrateBps),
1817 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001818
Sergey Silkincd02eba2020-01-20 14:48:40 +01001819 const uint32_t kMinEncBitrateKbps = 100;
1820 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001821 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001822 /*frame_size_pixels=*/codec_width_ * codec_height_,
1823 /*min_start_bitrate_bps=*/0,
1824 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1825 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001826 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1827
Sergey Silkincd02eba2020-01-20 14:48:40 +01001828 VideoEncoderConfig video_encoder_config;
1829 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1830 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1831 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1832 (kMinEncBitrateKbps + 1) * 1000;
1833 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1834 kMaxPayloadLength);
1835
1836 // When both encoder and app provide bitrate limits, the intersection of
1837 // provided sets should be used.
1838 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1839 WaitForEncodedFrame(1);
1840 EXPECT_EQ(kMaxEncBitrateKbps,
1841 bitrate_allocator_factory_.codec_config().maxBitrate);
1842 EXPECT_EQ(kMinEncBitrateKbps + 1,
1843 bitrate_allocator_factory_.codec_config().minBitrate);
1844
1845 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1846 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1847 (kMinEncBitrateKbps - 1) * 1000;
1848 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1849 kMaxPayloadLength);
1850 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001851 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001852 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001853 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001854 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001855 bitrate_allocator_factory_.codec_config().minBitrate);
1856
Sergey Silkincd02eba2020-01-20 14:48:40 +01001857 video_stream_encoder_->Stop();
1858}
1859
1860TEST_F(VideoStreamEncoderTest,
1861 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001863 DataRate::BitsPerSec(kTargetBitrateBps),
1864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001866
1867 const uint32_t kMinAppBitrateKbps = 100;
1868 const uint32_t kMaxAppBitrateKbps = 200;
1869 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1870 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1871 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1872 /*frame_size_pixels=*/codec_width_ * codec_height_,
1873 /*min_start_bitrate_bps=*/0,
1874 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1875 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1876 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1877
1878 VideoEncoderConfig video_encoder_config;
1879 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1880 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1881 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1882 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1884 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001885
Sergey Silkincd02eba2020-01-20 14:48:40 +01001886 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1887 WaitForEncodedFrame(1);
1888 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001889 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001890 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001891 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001892
1893 video_stream_encoder_->Stop();
1894}
1895
1896TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001897 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001899 DataRate::BitsPerSec(kTargetBitrateBps),
1900 DataRate::BitsPerSec(kTargetBitrateBps),
1901 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001902
1903 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001904 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001905 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001906 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001907 fake_encoder_.SetResolutionBitrateLimits(
1908 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1909
1910 VideoEncoderConfig video_encoder_config;
1911 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1912 video_encoder_config.max_bitrate_bps = 0;
1913 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1914 kMaxPayloadLength);
1915
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001916 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001917 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1918 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001919 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1920 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001921 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1922 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1923
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001924 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001925 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1926 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001927 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1928 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001929 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1930 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1931
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001932 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001933 // encoder for 360p should be used.
1934 video_source_.IncomingCapturedFrame(
1935 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1936 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001937 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1938 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001939 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1940 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1941
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001943 // ignored.
1944 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1945 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001946 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001948 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1949 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001950 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1951 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001952 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1953 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1954
1955 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1956 // for 270p should be used.
1957 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1958 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001959 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1960 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001961 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1962 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1963
1964 video_stream_encoder_->Stop();
1965}
1966
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001967TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001969 DataRate::BitsPerSec(kTargetBitrateBps),
1970 DataRate::BitsPerSec(kTargetBitrateBps),
1971 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001972
1973 VideoEncoderConfig video_encoder_config;
1974 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1975 video_encoder_config.max_bitrate_bps = 0;
1976 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1977 kMaxPayloadLength);
1978
1979 // Encode 720p frame to get the default encoder target bitrate.
1980 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1981 WaitForEncodedFrame(1);
1982 const uint32_t kDefaultTargetBitrateFor720pKbps =
1983 bitrate_allocator_factory_.codec_config()
1984 .simulcastStream[0]
1985 .targetBitrate;
1986
1987 // Set the max recommended encoder bitrate to something lower than the default
1988 // target bitrate.
1989 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1990 1280 * 720, 10 * 1000, 10 * 1000,
1991 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1992 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1993
1994 // Change resolution to trigger encoder reinitialization.
1995 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1996 WaitForEncodedFrame(2);
1997 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1998 WaitForEncodedFrame(3);
1999
2000 // Ensure the target bitrate is capped by the max bitrate.
2001 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2002 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2003 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2004 .simulcastStream[0]
2005 .targetBitrate *
2006 1000,
2007 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2008
2009 video_stream_encoder_->Stop();
2010}
2011
mflodmancc3d4422017-08-03 08:27:51 -07002012TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002013 EXPECT_TRUE(video_source_.has_sinks());
2014 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002015 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002016 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002017 EXPECT_FALSE(video_source_.has_sinks());
2018 EXPECT_TRUE(new_video_source.has_sinks());
2019
mflodmancc3d4422017-08-03 08:27:51 -07002020 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002021}
2022
mflodmancc3d4422017-08-03 08:27:51 -07002023TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002024 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002025 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002026 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002027 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002028}
2029
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002030class ResolutionAlignmentTest
2031 : public VideoStreamEncoderTest,
2032 public ::testing::WithParamInterface<
2033 ::testing::tuple<int, std::vector<double>>> {
2034 public:
2035 ResolutionAlignmentTest()
2036 : requested_alignment_(::testing::get<0>(GetParam())),
2037 scale_factors_(::testing::get<1>(GetParam())) {}
2038
2039 protected:
2040 const int requested_alignment_;
2041 const std::vector<double> scale_factors_;
2042};
2043
2044INSTANTIATE_TEST_SUITE_P(
2045 AlignmentAndScaleFactors,
2046 ResolutionAlignmentTest,
2047 ::testing::Combine(
2048 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2049 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2050 std::vector<double>{-1.0, -1.0},
2051 std::vector<double>{-1.0, -1.0, -1.0},
2052 std::vector<double>{4.0, 2.0, 1.0},
2053 std::vector<double>{9999.0, -1.0, 1.0},
2054 std::vector<double>{3.99, 2.01, 1.0},
2055 std::vector<double>{4.9, 1.7, 1.25},
2056 std::vector<double>{10.0, 4.0, 3.0},
2057 std::vector<double>{1.75, 3.5},
2058 std::vector<double>{1.5, 2.5},
2059 std::vector<double>{1.3, 1.0})));
2060
2061TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2062 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002063 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002064 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2065 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2066
2067 // Fill config with the scaling factor by which to reduce encoding size.
2068 const int num_streams = scale_factors_.size();
2069 VideoEncoderConfig config;
2070 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2071 for (int i = 0; i < num_streams; ++i) {
2072 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2073 }
2074 config.video_stream_factory =
2075 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2076 "VP8", /*max qp*/ 56, /*screencast*/ false,
2077 /*screenshare enabled*/ false);
2078 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2079
Henrik Boström381d1092020-05-12 18:49:07 +02002080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002081 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2082 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2083 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2084 // Wait for all layers before triggering event.
2085 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002086
2087 // On the 1st frame, we should have initialized the encoder and
2088 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002089 int64_t timestamp_ms = kFrameIntervalMs;
2090 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2091 WaitForEncodedFrame(timestamp_ms);
2092 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002093
2094 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2095 // (It's up the to the encoder to potentially drop the previous frame,
2096 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002097 timestamp_ms += kFrameIntervalMs;
2098 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2099 WaitForEncodedFrame(timestamp_ms);
2100 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2101
2102 VideoCodec codec = fake_encoder_.video_codec();
2103 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2104 // Frame size should be a multiple of the requested alignment.
2105 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2106 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2107 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2108 // Aspect ratio should match.
2109 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2110 codec.height * codec.simulcastStream[i].width);
2111 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002112
2113 video_stream_encoder_->Stop();
2114}
2115
Jonathan Yubc771b72017-12-08 17:04:29 -08002116TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2117 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002118 const int kWidth = 1280;
2119 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002120
2121 // We rely on the automatic resolution adaptation, but we handle framerate
2122 // adaptation manually by mocking the stats proxy.
2123 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002124
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002125 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002127 DataRate::BitsPerSec(kTargetBitrateBps),
2128 DataRate::BitsPerSec(kTargetBitrateBps),
2129 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002130 video_stream_encoder_->SetSource(&video_source_,
2131 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002132 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002133 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002134 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002135 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2136
Jonathan Yubc771b72017-12-08 17:04:29 -08002137 // Adapt down as far as possible.
2138 rtc::VideoSinkWants last_wants;
2139 int64_t t = 1;
2140 int loop_count = 0;
2141 do {
2142 ++loop_count;
2143 last_wants = video_source_.sink_wants();
2144
2145 // Simulate the framerate we've been asked to adapt to.
2146 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2147 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2148 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2149 mock_stats.input_frame_rate = fps;
2150 stats_proxy_->SetMockStats(mock_stats);
2151
2152 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2153 sink_.WaitForEncodedFrame(t);
2154 t += frame_interval_ms;
2155
mflodmancc3d4422017-08-03 08:27:51 -07002156 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002157 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002158 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002159 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2160 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002161 } while (video_source_.sink_wants().max_pixel_count <
2162 last_wants.max_pixel_count ||
2163 video_source_.sink_wants().max_framerate_fps <
2164 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002165
Jonathan Yubc771b72017-12-08 17:04:29 -08002166 // Verify that we've adapted all the way down.
2167 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002168 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002169 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2170 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002171 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002172 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2173 *video_source_.last_sent_height());
2174 EXPECT_EQ(kMinBalancedFramerateFps,
2175 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002176
Jonathan Yubc771b72017-12-08 17:04:29 -08002177 // Adapt back up the same number of times we adapted down.
2178 for (int i = 0; i < loop_count - 1; ++i) {
2179 last_wants = video_source_.sink_wants();
2180
2181 // Simulate the framerate we've been asked to adapt to.
2182 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2183 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2184 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2185 mock_stats.input_frame_rate = fps;
2186 stats_proxy_->SetMockStats(mock_stats);
2187
2188 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2189 sink_.WaitForEncodedFrame(t);
2190 t += frame_interval_ms;
2191
Henrik Boström91aa7322020-04-28 12:24:33 +02002192 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002193 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002194 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002195 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2196 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002197 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2198 last_wants.max_pixel_count ||
2199 video_source_.sink_wants().max_framerate_fps >
2200 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002201 }
2202
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002203 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002204 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002205 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002206 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2207 EXPECT_EQ((loop_count - 1) * 2,
2208 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002209
mflodmancc3d4422017-08-03 08:27:51 -07002210 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002211}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002212
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002213TEST_F(VideoStreamEncoderTest,
2214 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2215 video_stream_encoder_->OnBitrateUpdated(
2216 DataRate::BitsPerSec(kTargetBitrateBps),
2217 DataRate::BitsPerSec(kTargetBitrateBps),
2218 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002219 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002220
2221 const int kFrameWidth = 1280;
2222 const int kFrameHeight = 720;
2223
2224 int64_t ntp_time = kFrameIntervalMs;
2225
2226 // Force an input frame rate to be available, or the adaptation call won't
2227 // know what framerate to adapt form.
2228 const int kInputFps = 30;
2229 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2230 stats.input_frame_rate = kInputFps;
2231 stats_proxy_->SetMockStats(stats);
2232
2233 video_source_.set_adaptation_enabled(true);
2234 video_stream_encoder_->SetSource(
2235 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002236 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002237 video_source_.IncomingCapturedFrame(
2238 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2239 sink_.WaitForEncodedFrame(ntp_time);
2240 ntp_time += kFrameIntervalMs;
2241
2242 // Trigger CPU overuse.
2243 video_stream_encoder_->TriggerCpuOveruse();
2244 video_source_.IncomingCapturedFrame(
2245 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2246 sink_.WaitForEncodedFrame(ntp_time);
2247 ntp_time += kFrameIntervalMs;
2248
2249 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2250 EXPECT_EQ(std::numeric_limits<int>::max(),
2251 video_source_.sink_wants().max_pixel_count);
2252 // Some framerate constraint should be set.
2253 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2254 EXPECT_LT(restricted_fps, kInputFps);
2255 video_source_.IncomingCapturedFrame(
2256 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2257 sink_.WaitForEncodedFrame(ntp_time);
2258 ntp_time += 100;
2259
Henrik Boström2671dac2020-05-19 16:29:09 +02002260 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002261 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2262 // Give the encoder queue time to process the change in degradation preference
2263 // by waiting for an encoded frame.
2264 video_source_.IncomingCapturedFrame(
2265 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2266 sink_.WaitForEncodedFrame(ntp_time);
2267 ntp_time += kFrameIntervalMs;
2268
2269 video_stream_encoder_->TriggerQualityLow();
2270 video_source_.IncomingCapturedFrame(
2271 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2272 sink_.WaitForEncodedFrame(ntp_time);
2273 ntp_time += kFrameIntervalMs;
2274
2275 // Some resolution constraint should be set.
2276 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2277 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2278 kFrameWidth * kFrameHeight);
2279 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2280
2281 int pixel_count = video_source_.sink_wants().max_pixel_count;
2282 // Triggering a CPU underuse should not change the sink wants since it has
2283 // not been overused for resolution since we changed degradation preference.
2284 video_stream_encoder_->TriggerCpuUnderuse();
2285 video_source_.IncomingCapturedFrame(
2286 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2287 sink_.WaitForEncodedFrame(ntp_time);
2288 ntp_time += kFrameIntervalMs;
2289 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2290 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2291
Evan Shrubsole64469032020-06-11 10:45:29 +02002292 // Change the degradation preference back. CPU underuse should not adapt since
2293 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002294 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002295 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2296 video_source_.IncomingCapturedFrame(
2297 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2298 sink_.WaitForEncodedFrame(ntp_time);
2299 ntp_time += 100;
2300 // Resolution adaptations is gone after changing degradation preference.
2301 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2302 EXPECT_EQ(std::numeric_limits<int>::max(),
2303 video_source_.sink_wants().max_pixel_count);
2304 // The fps adaptation from above is now back.
2305 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2306
2307 // Trigger CPU underuse.
2308 video_stream_encoder_->TriggerCpuUnderuse();
2309 video_source_.IncomingCapturedFrame(
2310 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2311 sink_.WaitForEncodedFrame(ntp_time);
2312 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002313 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2314
2315 // Trigger QP underuse, fps should return to normal.
2316 video_stream_encoder_->TriggerQualityHigh();
2317 video_source_.IncomingCapturedFrame(
2318 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2319 sink_.WaitForEncodedFrame(ntp_time);
2320 ntp_time += kFrameIntervalMs;
2321 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002322
2323 video_stream_encoder_->Stop();
2324}
2325
mflodmancc3d4422017-08-03 08:27:51 -07002326TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002327 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002328 DataRate::BitsPerSec(kTargetBitrateBps),
2329 DataRate::BitsPerSec(kTargetBitrateBps),
2330 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002331 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002332
sprangc5d62e22017-04-02 23:53:04 -07002333 const int kFrameWidth = 1280;
2334 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002335
Åsa Persson8c1bf952018-09-13 10:42:19 +02002336 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002337
kthelgason5e13d412016-12-01 03:59:51 -08002338 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002339 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002340 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002341 frame_timestamp += kFrameIntervalMs;
2342
perkj803d97f2016-11-01 11:45:46 -07002343 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002344 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002345 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002346 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002347 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002348 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002349
asapersson0944a802017-04-07 00:57:58 -07002350 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002351 // wanted resolution.
2352 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2353 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2354 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002355 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002356
2357 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002358 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002359 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002360 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002361 // Give the encoder queue time to process the change in degradation preference
2362 // by waiting for an encoded frame.
2363 new_video_source.IncomingCapturedFrame(
2364 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2365 sink_.WaitForEncodedFrame(frame_timestamp);
2366 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002367 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002368 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002369
sprangc5d62e22017-04-02 23:53:04 -07002370 // Force an input frame rate to be available, or the adaptation call won't
2371 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002372 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002373 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002374 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002375 stats_proxy_->SetMockStats(stats);
2376
mflodmancc3d4422017-08-03 08:27:51 -07002377 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002378 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002379 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002380 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002381 frame_timestamp += kFrameIntervalMs;
2382
2383 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002384 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002385 EXPECT_EQ(std::numeric_limits<int>::max(),
2386 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002387 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002388
asapersson02465b82017-04-10 01:12:52 -07002389 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002390 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2391 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002392 // Give the encoder queue time to process the change in degradation preference
2393 // by waiting for an encoded frame.
2394 new_video_source.IncomingCapturedFrame(
2395 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2396 sink_.WaitForEncodedFrame(frame_timestamp);
2397 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002398 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002399
mflodmancc3d4422017-08-03 08:27:51 -07002400 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002401 new_video_source.IncomingCapturedFrame(
2402 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002403 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002404 frame_timestamp += kFrameIntervalMs;
2405
2406 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002407 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002408
2409 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002410 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002411 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002412 // Give the encoder queue time to process the change in degradation preference
2413 // by waiting for an encoded frame.
2414 new_video_source.IncomingCapturedFrame(
2415 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2416 sink_.WaitForEncodedFrame(frame_timestamp);
2417 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002418 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2419 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002420 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002421 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002422
2423 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002424 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002425 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002426 // Give the encoder queue time to process the change in degradation preference
2427 // by waiting for an encoded frame.
2428 new_video_source.IncomingCapturedFrame(
2429 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2430 sink_.WaitForEncodedFrame(frame_timestamp);
2431 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002432 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2433 EXPECT_EQ(std::numeric_limits<int>::max(),
2434 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002435 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002436
mflodmancc3d4422017-08-03 08:27:51 -07002437 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002438}
2439
mflodmancc3d4422017-08-03 08:27:51 -07002440TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002442 DataRate::BitsPerSec(kTargetBitrateBps),
2443 DataRate::BitsPerSec(kTargetBitrateBps),
2444 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002445
asaperssonfab67072017-04-04 05:51:49 -07002446 const int kWidth = 1280;
2447 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002448 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002449 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002450 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2451 EXPECT_FALSE(stats.bw_limited_resolution);
2452 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2453
2454 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002455 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002456 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002457 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002458
2459 stats = stats_proxy_->GetStats();
2460 EXPECT_TRUE(stats.bw_limited_resolution);
2461 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2462
2463 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002464 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002465 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002466 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002467
2468 stats = stats_proxy_->GetStats();
2469 EXPECT_FALSE(stats.bw_limited_resolution);
2470 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2471 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2472
mflodmancc3d4422017-08-03 08:27:51 -07002473 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002474}
2475
mflodmancc3d4422017-08-03 08:27:51 -07002476TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002478 DataRate::BitsPerSec(kTargetBitrateBps),
2479 DataRate::BitsPerSec(kTargetBitrateBps),
2480 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002481
2482 const int kWidth = 1280;
2483 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002484 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002485 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002486 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2487 EXPECT_FALSE(stats.cpu_limited_resolution);
2488 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2489
2490 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002491 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002492 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002493 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002494
2495 stats = stats_proxy_->GetStats();
2496 EXPECT_TRUE(stats.cpu_limited_resolution);
2497 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2498
2499 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002500 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002501 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002502 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002503
2504 stats = stats_proxy_->GetStats();
2505 EXPECT_FALSE(stats.cpu_limited_resolution);
2506 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002507 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002508
mflodmancc3d4422017-08-03 08:27:51 -07002509 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002510}
2511
mflodmancc3d4422017-08-03 08:27:51 -07002512TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002514 DataRate::BitsPerSec(kTargetBitrateBps),
2515 DataRate::BitsPerSec(kTargetBitrateBps),
2516 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002517
asaperssonfab67072017-04-04 05:51:49 -07002518 const int kWidth = 1280;
2519 const int kHeight = 720;
2520 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002521 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002522 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002523 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002524 EXPECT_FALSE(stats.cpu_limited_resolution);
2525 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2526
asaperssonfab67072017-04-04 05:51:49 -07002527 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002528 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002529 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002530 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002531 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002532 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002533 EXPECT_TRUE(stats.cpu_limited_resolution);
2534 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2535
2536 // Set new source with adaptation still enabled.
2537 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002538 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002539 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002540
asaperssonfab67072017-04-04 05:51:49 -07002541 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002542 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002543 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002544 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002545 EXPECT_TRUE(stats.cpu_limited_resolution);
2546 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2547
2548 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002549 video_stream_encoder_->SetSource(&new_video_source,
2550 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002551
asaperssonfab67072017-04-04 05:51:49 -07002552 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002553 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002554 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002555 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002556 EXPECT_FALSE(stats.cpu_limited_resolution);
2557 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2558
2559 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002560 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002561 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002562
asaperssonfab67072017-04-04 05:51:49 -07002563 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002564 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002565 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002566 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002567 EXPECT_TRUE(stats.cpu_limited_resolution);
2568 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2569
asaperssonfab67072017-04-04 05:51:49 -07002570 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002571 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002572 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002573 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002574 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002575 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002576 EXPECT_FALSE(stats.cpu_limited_resolution);
2577 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002578 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002579
mflodmancc3d4422017-08-03 08:27:51 -07002580 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002581}
2582
mflodmancc3d4422017-08-03 08:27:51 -07002583TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002585 DataRate::BitsPerSec(kTargetBitrateBps),
2586 DataRate::BitsPerSec(kTargetBitrateBps),
2587 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002588
asaperssonfab67072017-04-04 05:51:49 -07002589 const int kWidth = 1280;
2590 const int kHeight = 720;
2591 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002592 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002593 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002594 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002595 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002596 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002597
2598 // Set new source with adaptation still enabled.
2599 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002600 video_stream_encoder_->SetSource(&new_video_source,
2601 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002602
asaperssonfab67072017-04-04 05:51:49 -07002603 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002604 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002605 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002606 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002607 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002608 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002609
asaperssonfab67072017-04-04 05:51:49 -07002610 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002611 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002612 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002613 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002614 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002615 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002616 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002617 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002618
asaperssonfab67072017-04-04 05:51:49 -07002619 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002620 video_stream_encoder_->SetSource(&new_video_source,
2621 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002622
asaperssonfab67072017-04-04 05:51:49 -07002623 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002624 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002625 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002626 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002627 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002628 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002629
asapersson02465b82017-04-10 01:12:52 -07002630 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002631 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002632 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002633
asaperssonfab67072017-04-04 05:51:49 -07002634 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002635 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002636 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002637 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002638 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002639 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2640 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002641
mflodmancc3d4422017-08-03 08:27:51 -07002642 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002643}
2644
mflodmancc3d4422017-08-03 08:27:51 -07002645TEST_F(VideoStreamEncoderTest,
2646 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002647 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002648 DataRate::BitsPerSec(kTargetBitrateBps),
2649 DataRate::BitsPerSec(kTargetBitrateBps),
2650 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002651
2652 const int kWidth = 1280;
2653 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002654 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002655 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002656 video_source_.IncomingCapturedFrame(
2657 CreateFrame(timestamp_ms, kWidth, kHeight));
2658 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002659 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2661 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2662
2663 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002664 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002665 timestamp_ms += kFrameIntervalMs;
2666 video_source_.IncomingCapturedFrame(
2667 CreateFrame(timestamp_ms, kWidth, kHeight));
2668 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002669 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2671 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2672
2673 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002674 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002675 timestamp_ms += kFrameIntervalMs;
2676 video_source_.IncomingCapturedFrame(
2677 CreateFrame(timestamp_ms, kWidth, kHeight));
2678 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002679 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2680 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2681 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2682
Niels Möller4db138e2018-04-19 09:04:13 +02002683 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002684 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002685
2686 VideoEncoderConfig video_encoder_config;
2687 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2688 // Make format different, to force recreation of encoder.
2689 video_encoder_config.video_format.parameters["foo"] = "foo";
2690 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002691 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002692 timestamp_ms += kFrameIntervalMs;
2693 video_source_.IncomingCapturedFrame(
2694 CreateFrame(timestamp_ms, kWidth, kHeight));
2695 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002696 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2697 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2698 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2699
mflodmancc3d4422017-08-03 08:27:51 -07002700 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002701}
2702
mflodmancc3d4422017-08-03 08:27:51 -07002703TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002704 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002705 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002706 DataRate::BitsPerSec(kTargetBitrateBps),
2707 DataRate::BitsPerSec(kTargetBitrateBps),
2708 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2709
2710 const int kWidth = 1280;
2711 const int kHeight = 720;
2712 int sequence = 1;
2713
2714 // Enable BALANCED preference, no initial limitation.
2715 test::FrameForwarder source;
2716 video_stream_encoder_->SetSource(&source,
2717 webrtc::DegradationPreference::BALANCED);
2718 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2719 WaitForEncodedFrame(sequence++);
2720 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2721 EXPECT_FALSE(stats.cpu_limited_resolution);
2722 EXPECT_FALSE(stats.cpu_limited_framerate);
2723 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2724
2725 // Trigger CPU overuse, should now adapt down.
2726 video_stream_encoder_->TriggerCpuOveruse();
2727 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2728 WaitForEncodedFrame(sequence++);
2729 stats = stats_proxy_->GetStats();
2730 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2731
2732 // Set new degradation preference should clear restrictions since we changed
2733 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002734 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002735 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2736 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2737 WaitForEncodedFrame(sequence++);
2738 stats = stats_proxy_->GetStats();
2739 EXPECT_FALSE(stats.cpu_limited_resolution);
2740 EXPECT_FALSE(stats.cpu_limited_framerate);
2741 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2742
2743 // Force an input frame rate to be available, or the adaptation call won't
2744 // know what framerate to adapt from.
2745 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2746 mock_stats.input_frame_rate = 30;
2747 stats_proxy_->SetMockStats(mock_stats);
2748 video_stream_encoder_->TriggerCpuOveruse();
2749 stats_proxy_->ResetMockStats();
2750 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2751 WaitForEncodedFrame(sequence++);
2752
2753 // We have now adapted once.
2754 stats = stats_proxy_->GetStats();
2755 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2756
2757 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002758 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2759 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002760 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2761 WaitForEncodedFrame(sequence++);
2762 stats = stats_proxy_->GetStats();
2763 EXPECT_FALSE(stats.cpu_limited_resolution);
2764 EXPECT_FALSE(stats.cpu_limited_framerate);
2765 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2766
2767 video_stream_encoder_->Stop();
2768}
2769
2770TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002771 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002773 DataRate::BitsPerSec(kTargetBitrateBps),
2774 DataRate::BitsPerSec(kTargetBitrateBps),
2775 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002776
asapersson0944a802017-04-07 00:57:58 -07002777 const int kWidth = 1280;
2778 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002779 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002780
asaperssonfab67072017-04-04 05:51:49 -07002781 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002782 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002783 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002784 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002785 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002786 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2787
asapersson02465b82017-04-10 01:12:52 -07002788 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002789 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002790 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002791 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002792 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002793 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002794 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002795 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2796
2797 // Set new source with adaptation still enabled.
2798 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002799 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002800 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002801
2802 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002803 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002804 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002805 stats = stats_proxy_->GetStats();
2806 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002807 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002808 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2809
sprangc5d62e22017-04-02 23:53:04 -07002810 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002811 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002812 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002813 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002814 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002815 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002816 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002817 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002818 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002819 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002820 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2821
sprangc5d62e22017-04-02 23:53:04 -07002822 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002823 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002824 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2825 mock_stats.input_frame_rate = 30;
2826 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002827 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002828 stats_proxy_->ResetMockStats();
2829
2830 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002831 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002832 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002833
2834 // Framerate now adapted.
2835 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002836 EXPECT_FALSE(stats.cpu_limited_resolution);
2837 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002838 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2839
2840 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002841 video_stream_encoder_->SetSource(&new_video_source,
2842 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002843 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002844 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002845 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002846
2847 stats = stats_proxy_->GetStats();
2848 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002849 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002850 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2851
2852 // Try to trigger overuse. Should not succeed.
2853 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002854 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002855 stats_proxy_->ResetMockStats();
2856
2857 stats = stats_proxy_->GetStats();
2858 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002859 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002860 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2861
2862 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002863 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002864 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002865 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002866 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002867 stats = stats_proxy_->GetStats();
2868 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002869 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002870 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002871
2872 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002873 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002874 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002875 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002876 stats = stats_proxy_->GetStats();
2877 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002878 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002879 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2880
2881 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002882 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002883 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002884 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002885 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002886 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002887 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002888 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002889 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002890 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002891 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2892
2893 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002894 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002895 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002896 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002897 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002898 stats = stats_proxy_->GetStats();
2899 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002900 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002901 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002902 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002903
mflodmancc3d4422017-08-03 08:27:51 -07002904 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002905}
2906
mflodmancc3d4422017-08-03 08:27:51 -07002907TEST_F(VideoStreamEncoderTest,
2908 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002909 const int kWidth = 1280;
2910 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002911 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002912 DataRate::BitsPerSec(kTargetBitrateBps),
2913 DataRate::BitsPerSec(kTargetBitrateBps),
2914 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002915
asaperssonfab67072017-04-04 05:51:49 -07002916 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002917 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002918
asaperssonfab67072017-04-04 05:51:49 -07002919 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002920 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002921
asaperssonfab67072017-04-04 05:51:49 -07002922 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002924
asaperssonfab67072017-04-04 05:51:49 -07002925 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002926 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002927
kthelgason876222f2016-11-29 01:44:11 -08002928 // Expect a scale down.
2929 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002930 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002931
asapersson02465b82017-04-10 01:12:52 -07002932 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002933 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002934 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002935 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002936
asaperssonfab67072017-04-04 05:51:49 -07002937 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002938 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002939 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002940 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002941
asaperssonfab67072017-04-04 05:51:49 -07002942 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002943 EXPECT_EQ(std::numeric_limits<int>::max(),
2944 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002945
asaperssonfab67072017-04-04 05:51:49 -07002946 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002947 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002948 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002949 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002950
asapersson02465b82017-04-10 01:12:52 -07002951 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002952 EXPECT_EQ(std::numeric_limits<int>::max(),
2953 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002954
mflodmancc3d4422017-08-03 08:27:51 -07002955 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002956}
2957
mflodmancc3d4422017-08-03 08:27:51 -07002958TEST_F(VideoStreamEncoderTest,
2959 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002960 const int kWidth = 1280;
2961 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002962 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002963 DataRate::BitsPerSec(kTargetBitrateBps),
2964 DataRate::BitsPerSec(kTargetBitrateBps),
2965 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002966
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002967 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002968 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002969 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002970 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002971
2972 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002973 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002974 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002975 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2976 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2977
2978 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002979 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002980 EXPECT_THAT(source.sink_wants(),
2981 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002982 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2983 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2984 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2985
2986 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002987 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002988 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2989 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2990 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2991
mflodmancc3d4422017-08-03 08:27:51 -07002992 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002993}
2994
mflodmancc3d4422017-08-03 08:27:51 -07002995TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002996 const int kWidth = 1280;
2997 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002999 DataRate::BitsPerSec(kTargetBitrateBps),
3000 DataRate::BitsPerSec(kTargetBitrateBps),
3001 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003002
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003003 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003004 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003005 video_stream_encoder_->SetSource(&source,
3006 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003007 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3008 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003009 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003010
3011 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003012 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003013 EXPECT_THAT(source.sink_wants(),
3014 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003015 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3016 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3017 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3018
3019 // Trigger adapt down for same input resolution, expect no change.
3020 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3021 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003022 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003023 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3024 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3025 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3026
3027 // Trigger adapt down for larger input resolution, expect no change.
3028 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3029 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003030 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003031 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3032 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3033 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3034
mflodmancc3d4422017-08-03 08:27:51 -07003035 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003036}
3037
mflodmancc3d4422017-08-03 08:27:51 -07003038TEST_F(VideoStreamEncoderTest,
3039 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003040 const int kWidth = 1280;
3041 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003043 DataRate::BitsPerSec(kTargetBitrateBps),
3044 DataRate::BitsPerSec(kTargetBitrateBps),
3045 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003046
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003047 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003048 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003049 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003050 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003051
3052 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003053 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003054 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003055 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3056 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3057
3058 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003059 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003060 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003061 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3062 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3063
mflodmancc3d4422017-08-03 08:27:51 -07003064 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003065}
3066
mflodmancc3d4422017-08-03 08:27:51 -07003067TEST_F(VideoStreamEncoderTest,
3068 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003069 const int kWidth = 1280;
3070 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003071 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003072 DataRate::BitsPerSec(kTargetBitrateBps),
3073 DataRate::BitsPerSec(kTargetBitrateBps),
3074 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003075
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003076 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003077 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003078 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003079 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003080
3081 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003082 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003083 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003084 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003085 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3086
3087 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003088 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003089 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003090 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003091 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3092
mflodmancc3d4422017-08-03 08:27:51 -07003093 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003094}
3095
mflodmancc3d4422017-08-03 08:27:51 -07003096TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003097 const int kWidth = 1280;
3098 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003100 DataRate::BitsPerSec(kTargetBitrateBps),
3101 DataRate::BitsPerSec(kTargetBitrateBps),
3102 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003103
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003104 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003105 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003106 video_stream_encoder_->SetSource(&source,
3107 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003108
3109 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3110 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003111 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003112 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3113 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3114 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3115
3116 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003117 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003118 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003119 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3120 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3121 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3122
mflodmancc3d4422017-08-03 08:27:51 -07003123 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003124}
3125
mflodmancc3d4422017-08-03 08:27:51 -07003126TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003127 const int kWidth = 1280;
3128 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003129 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003130 DataRate::BitsPerSec(kTargetBitrateBps),
3131 DataRate::BitsPerSec(kTargetBitrateBps),
3132 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003133
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003134 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003135 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003136 video_stream_encoder_->SetSource(&source,
3137 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003138
3139 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3140 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003141 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003142 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3143 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3144 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3145
3146 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003147 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003148 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003149 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3150 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3151 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3152
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003154}
3155
mflodmancc3d4422017-08-03 08:27:51 -07003156TEST_F(VideoStreamEncoderTest,
3157 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003158 const int kWidth = 1280;
3159 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003161 DataRate::BitsPerSec(kTargetBitrateBps),
3162 DataRate::BitsPerSec(kTargetBitrateBps),
3163 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003164
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003165 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003166 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003167 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003168 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003169 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003170
3171 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003172 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003173 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003174 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3175 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3176
3177 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003178 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003179 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003180 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003181 EXPECT_THAT(source.sink_wants(),
3182 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003183 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3184 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3185
3186 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003188 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003189 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3190 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3191 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3192
mflodmancc3d4422017-08-03 08:27:51 -07003193 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003194}
3195
mflodmancc3d4422017-08-03 08:27:51 -07003196TEST_F(VideoStreamEncoderTest,
3197 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003198 const int kWidth = 1280;
3199 const int kHeight = 720;
3200 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003201 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003202 DataRate::BitsPerSec(kTargetBitrateBps),
3203 DataRate::BitsPerSec(kTargetBitrateBps),
3204 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003205
3206 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3207 stats.input_frame_rate = kInputFps;
3208 stats_proxy_->SetMockStats(stats);
3209
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003210 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003211 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3212 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003213 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003214
3215 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003216 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003217 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3218 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003219 EXPECT_THAT(video_source_.sink_wants(),
3220 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003221
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003222 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003223 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003224 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003225 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003226 // Give the encoder queue time to process the change in degradation preference
3227 // by waiting for an encoded frame.
3228 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3229 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003230 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003231
3232 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003233 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003234 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3235 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003236 EXPECT_THAT(new_video_source.sink_wants(),
3237 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003238
3239 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003240 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003241 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003242
mflodmancc3d4422017-08-03 08:27:51 -07003243 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003244}
3245
mflodmancc3d4422017-08-03 08:27:51 -07003246TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003247 const int kWidth = 1280;
3248 const int kHeight = 720;
3249 const size_t kNumFrames = 10;
3250
Henrik Boström381d1092020-05-12 18:49:07 +02003251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003252 DataRate::BitsPerSec(kTargetBitrateBps),
3253 DataRate::BitsPerSec(kTargetBitrateBps),
3254 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003255
asaperssond0de2952017-04-21 01:47:31 -07003256 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003257 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003258 video_source_.set_adaptation_enabled(true);
3259
3260 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3261 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3262
3263 int downscales = 0;
3264 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003265 video_source_.IncomingCapturedFrame(
3266 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3267 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003268
asaperssonfab67072017-04-04 05:51:49 -07003269 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003270 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003271 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003272 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003273
3274 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3275 ++downscales;
3276
3277 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3278 EXPECT_EQ(downscales,
3279 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3280 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003281 }
mflodmancc3d4422017-08-03 08:27:51 -07003282 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003283}
3284
mflodmancc3d4422017-08-03 08:27:51 -07003285TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003286 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3287 const int kWidth = 1280;
3288 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003289 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003290 DataRate::BitsPerSec(kTargetBitrateBps),
3291 DataRate::BitsPerSec(kTargetBitrateBps),
3292 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003293
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003294 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003295 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003296 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003297 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003298 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003299
Åsa Persson8c1bf952018-09-13 10:42:19 +02003300 int64_t timestamp_ms = kFrameIntervalMs;
3301 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003302 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003303 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003304 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3305 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3306
3307 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003308 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003309 timestamp_ms += kFrameIntervalMs;
3310 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3311 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003312 EXPECT_THAT(source.sink_wants(),
3313 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003314 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3315 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3316
3317 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003318 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003319 timestamp_ms += kFrameIntervalMs;
3320 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003321 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003323 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3324 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3325
3326 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003327 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003328 timestamp_ms += kFrameIntervalMs;
3329 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3330 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003331 EXPECT_THAT(source.sink_wants(),
3332 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003333 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3334 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3335
3336 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003337 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003338 timestamp_ms += kFrameIntervalMs;
3339 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003340 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003341 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003342 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3343 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3344
mflodmancc3d4422017-08-03 08:27:51 -07003345 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003346}
3347
mflodmancc3d4422017-08-03 08:27:51 -07003348TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003349 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3350 const int kWidth = 1280;
3351 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003353 DataRate::BitsPerSec(kTargetBitrateBps),
3354 DataRate::BitsPerSec(kTargetBitrateBps),
3355 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003356
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003357 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003358 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003359 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003360 video_stream_encoder_->SetSource(&source,
3361 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003362
Åsa Persson8c1bf952018-09-13 10:42:19 +02003363 int64_t timestamp_ms = kFrameIntervalMs;
3364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003365 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003366 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003367 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3368 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3369
3370 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003371 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003372 timestamp_ms += kFrameIntervalMs;
3373 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3374 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003375 EXPECT_THAT(source.sink_wants(),
3376 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003377 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3378 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3379
3380 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003381 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003382 timestamp_ms += kFrameIntervalMs;
3383 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003384 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003385 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003386 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3387 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3388
3389 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003390 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003391 timestamp_ms += kFrameIntervalMs;
3392 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3393 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003394 EXPECT_THAT(source.sink_wants(),
3395 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003396 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3397 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3398
3399 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003400 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003401 timestamp_ms += kFrameIntervalMs;
3402 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003403 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003404 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003405 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3406 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3407
mflodmancc3d4422017-08-03 08:27:51 -07003408 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003409}
3410
Sergey Silkin41c650b2019-10-14 13:12:19 +02003411TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3412 fake_encoder_.SetResolutionBitrateLimits(
3413 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3414
Henrik Boström381d1092020-05-12 18:49:07 +02003415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003416 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3417 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3418 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3419 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003420
3421 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003422 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003423 source.set_adaptation_enabled(true);
3424 video_stream_encoder_->SetSource(
3425 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3426
3427 // Insert 720p frame.
3428 int64_t timestamp_ms = kFrameIntervalMs;
3429 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3430 WaitForEncodedFrame(1280, 720);
3431
3432 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003434 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3435 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3436 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3437 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003438 video_stream_encoder_->TriggerQualityLow();
3439
3440 // Insert 720p frame. It should be downscaled and encoded.
3441 timestamp_ms += kFrameIntervalMs;
3442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3443 WaitForEncodedFrame(960, 540);
3444
3445 // Trigger adapt up. Higher resolution should not be requested duo to lack
3446 // of bitrate.
3447 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003448 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003449
3450 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003452 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3453 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3454 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3455 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003456
3457 // Trigger adapt up. Higher resolution should be requested.
3458 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003459 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003460
3461 video_stream_encoder_->Stop();
3462}
3463
3464TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3465 fake_encoder_.SetResolutionBitrateLimits(
3466 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3467
3468 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003470 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3471 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3472 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3473 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003474
3475 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003476 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003477 source.set_adaptation_enabled(true);
3478 video_stream_encoder_->SetSource(
3479 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3480
3481 // Insert 720p frame. It should be dropped and lower resolution should be
3482 // requested.
3483 int64_t timestamp_ms = kFrameIntervalMs;
3484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3485 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003486 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003487
3488 // Insert 720p frame. It should be downscaled and encoded.
3489 timestamp_ms += kFrameIntervalMs;
3490 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3491 WaitForEncodedFrame(960, 540);
3492
3493 video_stream_encoder_->Stop();
3494}
3495
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003496class BalancedDegradationTest : public VideoStreamEncoderTest {
3497 protected:
3498 void SetupTest() {
3499 // Reset encoder for field trials to take effect.
3500 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003501 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003502
3503 // Enable BALANCED preference.
3504 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003505 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3506 }
3507
3508 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003510 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3511 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003512 }
3513
Åsa Persson45b176f2019-09-30 11:19:05 +02003514 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003515 timestamp_ms_ += kFrameIntervalMs;
3516 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003517 }
3518
3519 void InsertFrameAndWaitForEncoded() {
3520 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003521 sink_.WaitForEncodedFrame(timestamp_ms_);
3522 }
3523
3524 const int kWidth = 640; // pixels:640x360=230400
3525 const int kHeight = 360;
3526 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3527 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003528 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003529};
3530
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003531TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003532 test::ScopedFieldTrials field_trials(
3533 "WebRTC-Video-BalancedDegradationSettings/"
3534 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3535 SetupTest();
3536
3537 // Force input frame rate.
3538 const int kInputFps = 24;
3539 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3540 stats.input_frame_rate = kInputFps;
3541 stats_proxy_->SetMockStats(stats);
3542
Åsa Persson45b176f2019-09-30 11:19:05 +02003543 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003544 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003545
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003546 // Trigger adapt down, expect scaled down framerate and resolution,
3547 // since Fps diff (input-requested:0) < threshold.
3548 video_stream_encoder_->TriggerQualityLow();
3549 EXPECT_THAT(source_.sink_wants(),
3550 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003551
3552 video_stream_encoder_->Stop();
3553}
3554
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003555TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003556 test::ScopedFieldTrials field_trials(
3557 "WebRTC-Video-BalancedDegradationSettings/"
3558 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3559 SetupTest();
3560
3561 // Force input frame rate.
3562 const int kInputFps = 25;
3563 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3564 stats.input_frame_rate = kInputFps;
3565 stats_proxy_->SetMockStats(stats);
3566
Åsa Persson45b176f2019-09-30 11:19:05 +02003567 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003568 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003569
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003570 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3571 // Fps diff (input-requested:1) == threshold.
3572 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003573 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003574
3575 video_stream_encoder_->Stop();
3576}
3577
3578TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3579 test::ScopedFieldTrials field_trials(
3580 "WebRTC-Video-BalancedDegradationSettings/"
3581 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3582 SetupTest();
3583
3584 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3585
Åsa Persson45b176f2019-09-30 11:19:05 +02003586 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003587 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003588
3589 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3590 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003591 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003592
3593 video_stream_encoder_->Stop();
3594}
3595
Åsa Perssonccfb3402019-09-25 15:13:04 +02003596TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003597 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003598 "WebRTC-Video-BalancedDegradationSettings/"
3599 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003600 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003601
Åsa Persson1b247f12019-08-14 17:26:39 +02003602 const int kMinBitrateBps = 425000;
3603 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003604 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003605
Åsa Persson45b176f2019-09-30 11:19:05 +02003606 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003607 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003608 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3609
3610 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3611 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003612 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003613 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003614 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615
3616 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3617 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003618 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003619 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003620 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3621
Åsa Persson30ab0152019-08-27 12:22:33 +02003622 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3623 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003624 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003625 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003626 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003627 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3628
3629 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003630 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003631 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003632 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003633
Åsa Persson30ab0152019-08-27 12:22:33 +02003634 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003635 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003636 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003637 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003638 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003639 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3640
3641 video_stream_encoder_->Stop();
3642}
3643
Åsa Perssonccfb3402019-09-25 15:13:04 +02003644TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003645 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3646 test::ScopedFieldTrials field_trials(
3647 "WebRTC-Video-BalancedDegradationSettings/"
3648 "pixels:57600|129600|230400,fps:7|24|24/");
3649 SetupTest();
3650 OnBitrateUpdated(kLowTargetBitrateBps);
3651
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003652 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003653
3654 // Insert frame, expect scaled down:
3655 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3656 InsertFrame();
3657 EXPECT_FALSE(WaitForFrame(1000));
3658 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3659 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3660
3661 // Insert frame, expect scaled down:
3662 // resolution (320x180@24fps).
3663 InsertFrame();
3664 EXPECT_FALSE(WaitForFrame(1000));
3665 EXPECT_LT(source_.sink_wants().max_pixel_count,
3666 source_.last_wants().max_pixel_count);
3667 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3668
3669 // Frame should not be dropped (min pixels per frame reached).
3670 InsertFrameAndWaitForEncoded();
3671
3672 video_stream_encoder_->Stop();
3673}
3674
3675TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003676 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003677 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003678 "WebRTC-Video-BalancedDegradationSettings/"
3679 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003680 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003681
Åsa Persson30ab0152019-08-27 12:22:33 +02003682 const int kResolutionMinBitrateBps = 435000;
3683 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003684 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003685
Åsa Persson45b176f2019-09-30 11:19:05 +02003686 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003687 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003688 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3689
3690 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3691 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003692 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003693 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003694 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3695
3696 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3697 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003698 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003699 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003700 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3701
3702 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3703 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003704 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003705 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003706 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3707
Åsa Persson30ab0152019-08-27 12:22:33 +02003708 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3709 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003710 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003711 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003712 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3713
3714 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3715 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003716 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003717 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3718
3719 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003720 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003721 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003722 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003723 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003724 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3725
3726 video_stream_encoder_->Stop();
3727}
3728
Åsa Perssonccfb3402019-09-25 15:13:04 +02003729TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003730 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003731 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003732 "WebRTC-Video-BalancedDegradationSettings/"
3733 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003734 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003735
Åsa Persson30ab0152019-08-27 12:22:33 +02003736 const int kMinBitrateBps = 425000;
3737 const int kTooLowMinBitrateBps = 424000;
3738 const int kResolutionMinBitrateBps = 435000;
3739 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003740 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003741
Åsa Persson45b176f2019-09-30 11:19:05 +02003742 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003743 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003744 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3745
3746 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3747 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003748 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003749 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003750 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3751
3752 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3753 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003754 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003755 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003756 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3757
3758 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3759 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003760 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003761 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003762 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3763
3764 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3765 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003766 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003767 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3768
3769 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003770 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003771 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003772 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003773 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003774 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3775
3776 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003777 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003778 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003779 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003780 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3781
3782 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003783 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003784 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003785 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003786 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003787 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3788
Åsa Persson1b247f12019-08-14 17:26:39 +02003789 video_stream_encoder_->Stop();
3790}
3791
mflodmancc3d4422017-08-03 08:27:51 -07003792TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003793 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3794 const int kWidth = 1280;
3795 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003797 DataRate::BitsPerSec(kTargetBitrateBps),
3798 DataRate::BitsPerSec(kTargetBitrateBps),
3799 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003800
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003801 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003802 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003803 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003804 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003805 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003806
Åsa Persson8c1bf952018-09-13 10:42:19 +02003807 int64_t timestamp_ms = kFrameIntervalMs;
3808 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003809 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003810 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3812 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3813 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3814 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3815
3816 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003817 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003818 timestamp_ms += kFrameIntervalMs;
3819 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3820 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003821 EXPECT_THAT(source.sink_wants(),
3822 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003823 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3824 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3825 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3826 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3827
3828 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003829 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003830 timestamp_ms += kFrameIntervalMs;
3831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3832 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003833 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003834 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3835 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3836 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3837 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3838
Jonathan Yubc771b72017-12-08 17:04:29 -08003839 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003840 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003841 timestamp_ms += kFrameIntervalMs;
3842 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3843 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003844 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003845 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3846 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003847 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003848 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3849
Jonathan Yubc771b72017-12-08 17:04:29 -08003850 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003851 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003852 timestamp_ms += kFrameIntervalMs;
3853 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3854 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003855 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003856 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003857 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3858 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3859 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3861
Jonathan Yubc771b72017-12-08 17:04:29 -08003862 // Trigger quality adapt down, expect no change (min resolution reached).
3863 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003864 timestamp_ms += kFrameIntervalMs;
3865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3866 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003867 EXPECT_THAT(source.sink_wants(), FpsMax());
3868 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003869 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3870 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3871 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3872 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3873
Evan Shrubsole64469032020-06-11 10:45:29 +02003874 // Trigger quality adapt up, expect upscaled resolution (480x270).
3875 video_stream_encoder_->TriggerQualityHigh();
3876 timestamp_ms += kFrameIntervalMs;
3877 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3878 WaitForEncodedFrame(timestamp_ms);
3879 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3880 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3881 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3882 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3883 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3884
3885 // Trigger quality and cpu adapt up since both are most limited, expect
3886 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003887 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003888 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003889 timestamp_ms += kFrameIntervalMs;
3890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3891 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003892 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003893 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3894 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3895 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003896 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003897
Evan Shrubsole64469032020-06-11 10:45:29 +02003898 // Trigger quality and cpu adapt up since both are most limited, expect
3899 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003900 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003901 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003902 timestamp_ms += kFrameIntervalMs;
3903 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3904 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003905 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003906 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003907 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003908 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003909 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3910 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003911
Evan Shrubsole64469032020-06-11 10:45:29 +02003912 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3913 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003914 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003915 timestamp_ms += kFrameIntervalMs;
3916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3917 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003918 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003919 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003921 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003922 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003923
3924 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003926 timestamp_ms += kFrameIntervalMs;
3927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003928 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003929 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003930 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003931 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3932 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003933 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003934 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003935
mflodmancc3d4422017-08-03 08:27:51 -07003936 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003937}
3938
mflodmancc3d4422017-08-03 08:27:51 -07003939TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003940 const int kWidth = 640;
3941 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003942
Henrik Boström381d1092020-05-12 18:49:07 +02003943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003944 DataRate::BitsPerSec(kTargetBitrateBps),
3945 DataRate::BitsPerSec(kTargetBitrateBps),
3946 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003947
perkj803d97f2016-11-01 11:45:46 -07003948 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003949 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003950 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003951 }
3952
mflodmancc3d4422017-08-03 08:27:51 -07003953 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003954 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003955 video_source_.IncomingCapturedFrame(CreateFrame(
3956 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003957 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003958 }
3959
mflodmancc3d4422017-08-03 08:27:51 -07003960 video_stream_encoder_->Stop();
3961 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003962 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003963
Ying Wangef3998f2019-12-09 13:06:53 +01003964 EXPECT_METRIC_EQ(
3965 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3966 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003967 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3968}
3969
mflodmancc3d4422017-08-03 08:27:51 -07003970TEST_F(VideoStreamEncoderTest,
3971 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003972 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003973 DataRate::BitsPerSec(kTargetBitrateBps),
3974 DataRate::BitsPerSec(kTargetBitrateBps),
3975 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003976 const int kWidth = 640;
3977 const int kHeight = 360;
3978
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003979 video_stream_encoder_->SetSource(&video_source_,
3980 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003981
3982 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3983 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003984 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003985 }
3986
mflodmancc3d4422017-08-03 08:27:51 -07003987 video_stream_encoder_->Stop();
3988 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003989 stats_proxy_.reset();
3990
3991 EXPECT_EQ(0,
3992 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3993}
3994
Per Kjellanderdcef6412020-10-07 15:09:05 +02003995TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
3996 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01003997 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02003998 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08003999
4000 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004001 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004002 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004003 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4004 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004005
Henrik Boström381d1092020-05-12 18:49:07 +02004006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004007 DataRate::BitsPerSec(kLowTargetBitrateBps),
4008 DataRate::BitsPerSec(kLowTargetBitrateBps),
4009 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004010
sprang57c2fff2017-01-16 06:24:02 -08004011 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004012 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4013 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004014 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4015 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4016
Erik Språngd7329ca2019-02-21 21:19:53 +01004017 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004018 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004019 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004020
Per Kjellanderdcef6412020-10-07 15:09:05 +02004021 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004022 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004023 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4024 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004025 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004026 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004027
Per Kjellanderdcef6412020-10-07 15:09:05 +02004028 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004029 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004030 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004031 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004032 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4033 WaitForEncodedFrame(CurrentTimeMs());
4034 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004035 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004036 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004037
mflodmancc3d4422017-08-03 08:27:51 -07004038 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004039}
4040
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004041TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004042 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004043 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004044 kVideoLayersAllocation);
4045
4046 const int kDefaultFps = 30;
4047
4048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4049 DataRate::BitsPerSec(kLowTargetBitrateBps),
4050 DataRate::BitsPerSec(kLowTargetBitrateBps),
4051 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4052
4053 video_source_.IncomingCapturedFrame(
4054 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4055 WaitForEncodedFrame(CurrentTimeMs());
4056 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4057 VideoLayersAllocation last_layer_allocation =
4058 sink_.GetLastVideoLayersAllocation();
4059 // kLowTargetBitrateBps is only enough for one spatial layer.
4060 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4061
4062 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004063 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004064 // Check that encoder has been updated too, not just allocation observer.
4065 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4066 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4067
Erik Språng9d69cbe2020-10-22 17:44:42 +02004068 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004069 int number_of_layers_allocation = 1;
4070 const int64_t start_time_ms = CurrentTimeMs();
4071 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4072 video_source_.IncomingCapturedFrame(
4073 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4074 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004075 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4076 number_of_layers_allocation = sink_.number_of_layers_allocations();
4077 VideoLayersAllocation new_allocation =
4078 sink_.GetLastVideoLayersAllocation();
4079 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4080 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4081 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4082 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4083 .target_bitrate_per_temporal_layer,
4084 last_layer_allocation.active_spatial_layers[0]
4085 .target_bitrate_per_temporal_layer);
4086 last_layer_allocation = new_allocation;
4087 }
4088 }
4089 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4090 video_stream_encoder_->Stop();
4091}
4092
4093TEST_F(VideoStreamEncoderTest,
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004094 ReportsVideoLayersAllocationForVP8WithMidleLayerDisabled) {
4095 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4096 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4097 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004098 VideoEncoderConfig video_encoder_config;
4099 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4100 /* num_streams*/ 3, &video_encoder_config);
4101 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4102 video_encoder_config.content_type =
4103 VideoEncoderConfig::ContentType::kRealtimeVideo;
4104 video_encoder_config.encoder_specific_settings =
4105 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4106 VideoEncoder::GetDefaultVp8Settings());
4107 for (auto& layer : video_encoder_config.simulcast_layers) {
4108 layer.num_temporal_layers = 2;
4109 }
4110 // Simulcast layers are used for enabling/disabling streams.
4111 video_encoder_config.simulcast_layers[0].active = true;
4112 video_encoder_config.simulcast_layers[1].active = false;
4113 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004114 ConfigureEncoder(std::move(video_encoder_config),
4115 VideoStreamEncoder::BitrateAllocationCallbackType::
4116 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004117
4118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4119 DataRate::BitsPerSec(kTargetBitrateBps),
4120 DataRate::BitsPerSec(kTargetBitrateBps),
4121 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4122
4123 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4124 WaitForEncodedFrame(CurrentTimeMs());
4125 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4126 VideoLayersAllocation last_layer_allocation =
4127 sink_.GetLastVideoLayersAllocation();
4128
4129 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4130 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4131 .target_bitrate_per_temporal_layer,
4132 SizeIs(2));
4133 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4134 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4135 video_stream_encoder_->Stop();
4136}
4137
4138TEST_F(VideoStreamEncoderTest,
4139 ReportsVideoLayersAllocationForVP8WithMidleAndHighestLayerDisabled) {
4140 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4141 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4142 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004143 VideoEncoderConfig video_encoder_config;
4144 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4145 /* num_streams*/ 3, &video_encoder_config);
4146 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4147 video_encoder_config.content_type =
4148 VideoEncoderConfig::ContentType::kRealtimeVideo;
4149 video_encoder_config.encoder_specific_settings =
4150 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4151 VideoEncoder::GetDefaultVp8Settings());
4152 for (auto& layer : video_encoder_config.simulcast_layers) {
4153 layer.num_temporal_layers = 2;
4154 }
4155 // Simulcast layers are used for enabling/disabling streams.
4156 video_encoder_config.simulcast_layers[0].active = true;
4157 video_encoder_config.simulcast_layers[1].active = false;
4158 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004159 ConfigureEncoder(std::move(video_encoder_config),
4160 VideoStreamEncoder::BitrateAllocationCallbackType::
4161 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004162
4163 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4164 DataRate::BitsPerSec(kTargetBitrateBps),
4165 DataRate::BitsPerSec(kTargetBitrateBps),
4166 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4167
4168 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4169 WaitForEncodedFrame(CurrentTimeMs());
4170 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4171 VideoLayersAllocation last_layer_allocation =
4172 sink_.GetLastVideoLayersAllocation();
4173
4174 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4175 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4176 .target_bitrate_per_temporal_layer,
4177 SizeIs(2));
4178 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4179
4180 video_stream_encoder_->Stop();
4181}
4182
4183TEST_F(VideoStreamEncoderTest,
4184 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4185 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4186 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004187 VideoEncoderConfig video_encoder_config;
4188 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4189 /* num_streams*/ 1, &video_encoder_config);
4190 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4191 video_encoder_config.content_type =
4192 VideoEncoderConfig::ContentType::kRealtimeVideo;
4193 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4194 vp9_settings.numberOfSpatialLayers = 2;
4195 vp9_settings.numberOfTemporalLayers = 2;
4196 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4197 vp9_settings.automaticResizeOn = false;
4198 video_encoder_config.encoder_specific_settings =
4199 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4200 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004201 ConfigureEncoder(std::move(video_encoder_config),
4202 VideoStreamEncoder::BitrateAllocationCallbackType::
4203 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004204
4205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4206 DataRate::BitsPerSec(kTargetBitrateBps),
4207 DataRate::BitsPerSec(kTargetBitrateBps),
4208 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4209
4210 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4211 WaitForEncodedFrame(CurrentTimeMs());
4212 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4213 VideoLayersAllocation last_layer_allocation =
4214 sink_.GetLastVideoLayersAllocation();
4215
4216 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4217 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4218 .target_bitrate_per_temporal_layer,
4219 SizeIs(2));
4220 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4221 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4222 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4223 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4224 .target_bitrate_per_temporal_layer,
4225 SizeIs(2));
4226 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4227 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4228 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4229
4230 // Since full SVC is used, expect the top layer to utilize the full target
4231 // rate.
4232 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4233 .target_bitrate_per_temporal_layer[1],
4234 DataRate::BitsPerSec(kTargetBitrateBps));
4235 video_stream_encoder_->Stop();
4236}
4237
4238TEST_F(VideoStreamEncoderTest,
4239 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4240 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4241 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004242 VideoEncoderConfig video_encoder_config;
4243 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4244 /* num_streams*/ 1, &video_encoder_config);
4245 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4246 video_encoder_config.content_type =
4247 VideoEncoderConfig::ContentType::kRealtimeVideo;
4248 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4249 vp9_settings.numberOfSpatialLayers = 2;
4250 vp9_settings.numberOfTemporalLayers = 2;
4251 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4252 vp9_settings.automaticResizeOn = false;
4253 video_encoder_config.encoder_specific_settings =
4254 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4255 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004256 ConfigureEncoder(std::move(video_encoder_config),
4257 VideoStreamEncoder::BitrateAllocationCallbackType::
4258 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004259
4260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4261 DataRate::BitsPerSec(kTargetBitrateBps),
4262 DataRate::BitsPerSec(kTargetBitrateBps),
4263 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4264
4265 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4266 WaitForEncodedFrame(CurrentTimeMs());
4267 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4268 VideoLayersAllocation last_layer_allocation =
4269 sink_.GetLastVideoLayersAllocation();
4270
4271 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4272 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4273 .target_bitrate_per_temporal_layer,
4274 SizeIs(1));
4275 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4276 .target_bitrate_per_temporal_layer,
4277 SizeIs(1));
4278 // Since full SVC is used, expect the top layer to utilize the full target
4279 // rate.
4280 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4281 .target_bitrate_per_temporal_layer[0],
4282 DataRate::BitsPerSec(kTargetBitrateBps));
4283 video_stream_encoder_->Stop();
4284}
4285
4286TEST_F(VideoStreamEncoderTest,
4287 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4288 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4289 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004290 VideoEncoderConfig video_encoder_config;
4291 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4292 /* num_streams*/ 1, &video_encoder_config);
4293 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4294 video_encoder_config.content_type =
4295 VideoEncoderConfig::ContentType::kRealtimeVideo;
4296 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4297 vp9_settings.numberOfSpatialLayers = 2;
4298 vp9_settings.numberOfTemporalLayers = 2;
4299 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4300 vp9_settings.automaticResizeOn = false;
4301 video_encoder_config.encoder_specific_settings =
4302 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4303 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004304 ConfigureEncoder(std::move(video_encoder_config),
4305 VideoStreamEncoder::BitrateAllocationCallbackType::
4306 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004307
4308 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4309 DataRate::BitsPerSec(kTargetBitrateBps),
4310 DataRate::BitsPerSec(kTargetBitrateBps),
4311 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4312
4313 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4314 WaitForEncodedFrame(CurrentTimeMs());
4315 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4316 VideoLayersAllocation last_layer_allocation =
4317 sink_.GetLastVideoLayersAllocation();
4318
4319 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4320 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4321 .target_bitrate_per_temporal_layer,
4322 SizeIs(2));
4323 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4324 .target_bitrate_per_temporal_layer,
4325 SizeIs(2));
4326 // Since KSVC is, spatial layers are independend except on key frames.
4327 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4328 .target_bitrate_per_temporal_layer[1],
4329 DataRate::BitsPerSec(kTargetBitrateBps));
4330 video_stream_encoder_->Stop();
4331}
4332
4333TEST_F(VideoStreamEncoderTest,
4334 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4335 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4336 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4337 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004338 VideoEncoderConfig video_encoder_config;
4339 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4340 /* num_streams*/ 1, &video_encoder_config);
4341 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4342 video_encoder_config.content_type =
4343 VideoEncoderConfig::ContentType::kRealtimeVideo;
4344 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4345 vp9_settings.numberOfSpatialLayers = 3;
4346 vp9_settings.numberOfTemporalLayers = 2;
4347 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4348 vp9_settings.automaticResizeOn = false;
4349 video_encoder_config.encoder_specific_settings =
4350 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4351 vp9_settings);
4352 // Simulcast layers are used for enabling/disabling streams.
4353 video_encoder_config.simulcast_layers.resize(3);
4354 video_encoder_config.simulcast_layers[0].active = false;
4355 video_encoder_config.simulcast_layers[1].active = true;
4356 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004357 ConfigureEncoder(std::move(video_encoder_config),
4358 VideoStreamEncoder::BitrateAllocationCallbackType::
4359 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004360
4361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4362 DataRate::BitsPerSec(kTargetBitrateBps),
4363 DataRate::BitsPerSec(kTargetBitrateBps),
4364 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4365
4366 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4367 WaitForEncodedFrame(CurrentTimeMs());
4368 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4369 VideoLayersAllocation last_layer_allocation =
4370 sink_.GetLastVideoLayersAllocation();
4371
4372 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4373 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4374 .target_bitrate_per_temporal_layer,
4375 SizeIs(2));
4376 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4377 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4378
4379 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4380 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4381 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4382 .target_bitrate_per_temporal_layer,
4383 SizeIs(2));
4384 // Since full SVC is used, expect the top layer to utilize the full target
4385 // rate.
4386 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4387 .target_bitrate_per_temporal_layer[1],
4388 DataRate::BitsPerSec(kTargetBitrateBps));
4389 video_stream_encoder_->Stop();
4390}
4391
4392TEST_F(VideoStreamEncoderTest,
4393 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4394 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4395 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4396 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004397 VideoEncoderConfig video_encoder_config;
4398 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4399 /* num_streams*/ 1, &video_encoder_config);
4400 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4401 video_encoder_config.content_type =
4402 VideoEncoderConfig::ContentType::kRealtimeVideo;
4403 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4404 vp9_settings.numberOfSpatialLayers = 3;
4405 vp9_settings.numberOfTemporalLayers = 2;
4406 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4407 vp9_settings.automaticResizeOn = false;
4408 video_encoder_config.encoder_specific_settings =
4409 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4410 vp9_settings);
4411 // Simulcast layers are used for enabling/disabling streams.
4412 video_encoder_config.simulcast_layers.resize(3);
4413 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004414 ConfigureEncoder(std::move(video_encoder_config),
4415 VideoStreamEncoder::BitrateAllocationCallbackType::
4416 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004417
4418 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4419 DataRate::BitsPerSec(kTargetBitrateBps),
4420 DataRate::BitsPerSec(kTargetBitrateBps),
4421 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4422
4423 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4424 WaitForEncodedFrame(CurrentTimeMs());
4425 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4426 VideoLayersAllocation last_layer_allocation =
4427 sink_.GetLastVideoLayersAllocation();
4428
4429 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4430 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4431 .target_bitrate_per_temporal_layer,
4432 SizeIs(2));
4433 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4434 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4435
4436 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4437 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4438 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4439 .target_bitrate_per_temporal_layer,
4440 SizeIs(2));
4441 video_stream_encoder_->Stop();
4442}
4443
4444TEST_F(VideoStreamEncoderTest,
4445 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4446 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4447 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4448 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004449 VideoEncoderConfig video_encoder_config;
4450 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4451 /* num_streams*/ 1, &video_encoder_config);
4452 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4453 video_encoder_config.content_type =
4454 VideoEncoderConfig::ContentType::kRealtimeVideo;
4455 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4456 vp9_settings.numberOfSpatialLayers = 3;
4457 vp9_settings.numberOfTemporalLayers = 2;
4458 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4459 vp9_settings.automaticResizeOn = false;
4460 video_encoder_config.encoder_specific_settings =
4461 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4462 vp9_settings);
4463 // Simulcast layers are used for enabling/disabling streams.
4464 video_encoder_config.simulcast_layers.resize(3);
4465 video_encoder_config.simulcast_layers[0].active = false;
4466 video_encoder_config.simulcast_layers[1].active = false;
4467 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004468 ConfigureEncoder(std::move(video_encoder_config),
4469 VideoStreamEncoder::BitrateAllocationCallbackType::
4470 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004471
4472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4473 DataRate::BitsPerSec(kTargetBitrateBps),
4474 DataRate::BitsPerSec(kTargetBitrateBps),
4475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4476
4477 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4478 WaitForEncodedFrame(CurrentTimeMs());
4479 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4480 VideoLayersAllocation last_layer_allocation =
4481 sink_.GetLastVideoLayersAllocation();
4482
4483 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4484 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4485 .target_bitrate_per_temporal_layer,
4486 SizeIs(2));
4487 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4488 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4489 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4490 .target_bitrate_per_temporal_layer[1],
4491 DataRate::BitsPerSec(kTargetBitrateBps));
4492 video_stream_encoder_->Stop();
4493}
4494
4495TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4496 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004497 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004498 kVideoLayersAllocation);
4499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4500 DataRate::BitsPerSec(kTargetBitrateBps),
4501 DataRate::BitsPerSec(kTargetBitrateBps),
4502 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4503
4504 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4505 WaitForEncodedFrame(CurrentTimeMs());
4506 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4507 VideoLayersAllocation last_layer_allocation =
4508 sink_.GetLastVideoLayersAllocation();
4509
4510 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4511 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4512 .target_bitrate_per_temporal_layer,
4513 SizeIs(1));
4514 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4515 .target_bitrate_per_temporal_layer[0],
4516 DataRate::BitsPerSec(kTargetBitrateBps));
4517 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4518 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4519 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4520 video_stream_encoder_->Stop();
4521}
4522
4523TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004524 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4525 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004526 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004527 kVideoLayersAllocation);
4528
4529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4530 DataRate::BitsPerSec(kLowTargetBitrateBps),
4531 DataRate::BitsPerSec(kLowTargetBitrateBps),
4532 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4533
4534 video_source_.IncomingCapturedFrame(
4535 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4536 WaitForEncodedFrame(CurrentTimeMs());
4537 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4538 VideoLayersAllocation last_layer_allocation =
4539 sink_.GetLastVideoLayersAllocation();
4540 // kLowTargetBitrateBps is only enough for one spatial layer.
4541 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4542 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4543 .target_bitrate_per_temporal_layer[0],
4544 DataRate::BitsPerSec(kLowTargetBitrateBps));
4545
4546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4547 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4548 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4549 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4550 video_source_.IncomingCapturedFrame(
4551 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4552 WaitForEncodedFrame(CurrentTimeMs());
4553
4554 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4555 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4556 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4557 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4558 .target_bitrate_per_temporal_layer[0],
4559 DataRate::Zero());
4560
4561 video_stream_encoder_->Stop();
4562}
4563
Per Kjellander4190ce92020-12-15 17:24:55 +01004564TEST_F(VideoStreamEncoderTest,
4565 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4566 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004567 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004568 kVideoLayersAllocation);
4569
4570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4571 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4572 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4573 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4574
4575 video_source_.IncomingCapturedFrame(
4576 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4577 WaitForEncodedFrame(CurrentTimeMs());
4578 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4579 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4580 SizeIs(2));
4581 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4582 codec_width_);
4583 EXPECT_EQ(
4584 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4585 codec_height_);
4586
4587 video_source_.IncomingCapturedFrame(
4588 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4589 WaitForEncodedFrame(CurrentTimeMs());
4590 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4591 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4592 SizeIs(2));
4593 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4594 codec_width_ / 2);
4595 EXPECT_EQ(
4596 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4597 codec_height_ / 2);
4598
4599 video_stream_encoder_->Stop();
4600}
4601
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004602TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4603 // 2 TLs configured, temporal layers supported by encoder.
4604 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004605 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004606 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004607 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004608 fake_encoder_.SetTemporalLayersSupported(0, true);
4609
4610 // Bitrate allocated across temporal layers.
4611 const int kTl0Bps = kTargetBitrateBps *
4612 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004613 kNumTemporalLayers, /*temporal_id*/ 0,
4614 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004615 const int kTl1Bps = kTargetBitrateBps *
4616 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004617 kNumTemporalLayers, /*temporal_id*/ 1,
4618 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004619 VideoBitrateAllocation expected_bitrate;
4620 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4621 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4622
4623 VerifyAllocatedBitrate(expected_bitrate);
4624 video_stream_encoder_->Stop();
4625}
4626
4627TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4628 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004629 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004630 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004631 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004632 fake_encoder_.SetTemporalLayersSupported(0, false);
4633
4634 // Temporal layers not supported by the encoder.
4635 // Total bitrate should be at ti:0.
4636 VideoBitrateAllocation expected_bitrate;
4637 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4638
4639 VerifyAllocatedBitrate(expected_bitrate);
4640 video_stream_encoder_->Stop();
4641}
4642
4643TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004644 webrtc::test::ScopedFieldTrials field_trials(
4645 "WebRTC-Video-QualityScalerSettings/"
4646 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4647 // Reset encoder for field trials to take effect.
4648 ConfigureEncoder(video_encoder_config_.Copy());
4649
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004650 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004651 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004652 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004653 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004654 fake_encoder_.SetTemporalLayersSupported(0, true);
4655 fake_encoder_.SetTemporalLayersSupported(1, false);
4656
4657 const int kS0Bps = 150000;
4658 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004659 kS0Bps *
4660 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4661 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004662 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004663 kS0Bps *
4664 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4665 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004666 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4667 // Temporal layers not supported by si:1.
4668 VideoBitrateAllocation expected_bitrate;
4669 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4670 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4671 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4672
4673 VerifyAllocatedBitrate(expected_bitrate);
4674 video_stream_encoder_->Stop();
4675}
4676
Niels Möller7dc26b72017-12-06 10:27:48 +01004677TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4678 const int kFrameWidth = 1280;
4679 const int kFrameHeight = 720;
4680 const int kFramerate = 24;
4681
Henrik Boström381d1092020-05-12 18:49:07 +02004682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004683 DataRate::BitsPerSec(kTargetBitrateBps),
4684 DataRate::BitsPerSec(kTargetBitrateBps),
4685 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004686 test::FrameForwarder source;
4687 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004688 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004689
4690 // Insert a single frame, triggering initial configuration.
4691 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4692 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4693
4694 EXPECT_EQ(
4695 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4696 kDefaultFramerate);
4697
4698 // Trigger reconfigure encoder (without resetting the entire instance).
4699 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004700 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4701 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004702 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004703 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004704 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004705 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4706
4707 // Detector should be updated with fps limit from codec config.
4708 EXPECT_EQ(
4709 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4710 kFramerate);
4711
4712 // Trigger overuse, max framerate should be reduced.
4713 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4714 stats.input_frame_rate = kFramerate;
4715 stats_proxy_->SetMockStats(stats);
4716 video_stream_encoder_->TriggerCpuOveruse();
4717 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4718 int adapted_framerate =
4719 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4720 EXPECT_LT(adapted_framerate, kFramerate);
4721
4722 // Trigger underuse, max framerate should go back to codec configured fps.
4723 // Set extra low fps, to make sure it's actually reset, not just incremented.
4724 stats = stats_proxy_->GetStats();
4725 stats.input_frame_rate = adapted_framerate / 2;
4726 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004727 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004728 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4729 EXPECT_EQ(
4730 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4731 kFramerate);
4732
4733 video_stream_encoder_->Stop();
4734}
4735
4736TEST_F(VideoStreamEncoderTest,
4737 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
4738 const int kFrameWidth = 1280;
4739 const int kFrameHeight = 720;
4740 const int kLowFramerate = 15;
4741 const int kHighFramerate = 25;
4742
Henrik Boström381d1092020-05-12 18:49:07 +02004743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004744 DataRate::BitsPerSec(kTargetBitrateBps),
4745 DataRate::BitsPerSec(kTargetBitrateBps),
4746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004747 test::FrameForwarder source;
4748 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004749 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004750
4751 // Trigger initial configuration.
4752 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004753 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4754 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004755 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004756 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02004757 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02004758 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004759 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4760
4761 EXPECT_EQ(
4762 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4763 kLowFramerate);
4764
4765 // Trigger overuse, max framerate should be reduced.
4766 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4767 stats.input_frame_rate = kLowFramerate;
4768 stats_proxy_->SetMockStats(stats);
4769 video_stream_encoder_->TriggerCpuOveruse();
4770 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4771 int adapted_framerate =
4772 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4773 EXPECT_LT(adapted_framerate, kLowFramerate);
4774
4775 // Reconfigure the encoder with a new (higher max framerate), max fps should
4776 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02004777 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004778 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4779 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004780 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4782
4783 EXPECT_EQ(
4784 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4785 adapted_framerate);
4786
4787 // Trigger underuse, max framerate should go back to codec configured fps.
4788 stats = stats_proxy_->GetStats();
4789 stats.input_frame_rate = adapted_framerate;
4790 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004791 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004792 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4793 EXPECT_EQ(
4794 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4795 kHighFramerate);
4796
4797 video_stream_encoder_->Stop();
4798}
4799
mflodmancc3d4422017-08-03 08:27:51 -07004800TEST_F(VideoStreamEncoderTest,
4801 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07004802 const int kFrameWidth = 1280;
4803 const int kFrameHeight = 720;
4804 const int kFramerate = 24;
4805
Henrik Boström381d1092020-05-12 18:49:07 +02004806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004807 DataRate::BitsPerSec(kTargetBitrateBps),
4808 DataRate::BitsPerSec(kTargetBitrateBps),
4809 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07004810 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004811 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004812 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07004813
4814 // Trigger initial configuration.
4815 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004816 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4817 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07004818 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07004819 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004820 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004821 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004822 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004823
Niels Möller7dc26b72017-12-06 10:27:48 +01004824 EXPECT_EQ(
4825 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4826 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004827
4828 // Trigger overuse, max framerate should be reduced.
4829 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4830 stats.input_frame_rate = kFramerate;
4831 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004832 video_stream_encoder_->TriggerCpuOveruse();
4833 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004834 int adapted_framerate =
4835 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004836 EXPECT_LT(adapted_framerate, kFramerate);
4837
4838 // Change degradation preference to not enable framerate scaling. Target
4839 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004840 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004841 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004842 EXPECT_EQ(
4843 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4844 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004845
mflodmancc3d4422017-08-03 08:27:51 -07004846 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004847}
4848
mflodmancc3d4422017-08-03 08:27:51 -07004849TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004850 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004852 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4853 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4854 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004855 const int kWidth = 640;
4856 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004857
asaperssonfab67072017-04-04 05:51:49 -07004858 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004859
4860 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004861 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004862
4863 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004864 EXPECT_TRUE_WAIT(
4865 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004866
sprangc5d62e22017-04-02 23:53:04 -07004867 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004868
asaperssonfab67072017-04-04 05:51:49 -07004869 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004870 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004871 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004872
4873 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004874 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004875
Henrik Boström2671dac2020-05-19 16:29:09 +02004876 EXPECT_TRUE_WAIT(
4877 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004878
mflodmancc3d4422017-08-03 08:27:51 -07004879 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004880}
4881
mflodmancc3d4422017-08-03 08:27:51 -07004882TEST_F(VideoStreamEncoderTest,
4883 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004884 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004886 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4887 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4888 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004889 const int kWidth = 640;
4890 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004891
4892 // We expect the n initial frames to get dropped.
4893 int i;
4894 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004895 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004896 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004897 }
4898 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004899 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004900 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004901
4902 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004903 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004904
mflodmancc3d4422017-08-03 08:27:51 -07004905 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004906}
4907
mflodmancc3d4422017-08-03 08:27:51 -07004908TEST_F(VideoStreamEncoderTest,
4909 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004910 const int kWidth = 640;
4911 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004912 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004913 DataRate::BitsPerSec(kLowTargetBitrateBps),
4914 DataRate::BitsPerSec(kLowTargetBitrateBps),
4915 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004916
4917 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004918 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004919 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004920
asaperssonfab67072017-04-04 05:51:49 -07004921 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004922 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004923 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004924
mflodmancc3d4422017-08-03 08:27:51 -07004925 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004926}
4927
mflodmancc3d4422017-08-03 08:27:51 -07004928TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004929 const int kWidth = 640;
4930 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004931 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004932
4933 VideoEncoderConfig video_encoder_config;
4934 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4935 // Make format different, to force recreation of encoder.
4936 video_encoder_config.video_format.parameters["foo"] = "foo";
4937 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004938 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004940 DataRate::BitsPerSec(kLowTargetBitrateBps),
4941 DataRate::BitsPerSec(kLowTargetBitrateBps),
4942 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004943
kthelgasonb83797b2017-02-14 11:57:25 -08004944 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004945 video_stream_encoder_->SetSource(&video_source_,
4946 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004947
asaperssonfab67072017-04-04 05:51:49 -07004948 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004949 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004950 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004951
mflodmancc3d4422017-08-03 08:27:51 -07004952 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004953 fake_encoder_.SetQualityScaling(true);
4954}
4955
Åsa Persson139f4dc2019-08-02 09:29:58 +02004956TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4957 webrtc::test::ScopedFieldTrials field_trials(
4958 "WebRTC-Video-QualityScalerSettings/"
4959 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4960 // Reset encoder for field trials to take effect.
4961 ConfigureEncoder(video_encoder_config_.Copy());
4962 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4963 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4964 const int kWidth = 640;
4965 const int kHeight = 360;
4966
Henrik Boström381d1092020-05-12 18:49:07 +02004967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004968 DataRate::BitsPerSec(kTargetBitrateBps),
4969 DataRate::BitsPerSec(kTargetBitrateBps),
4970 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004971 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4972 // Frame should not be dropped.
4973 WaitForEncodedFrame(1);
4974
Henrik Boström381d1092020-05-12 18:49:07 +02004975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004976 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4977 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4978 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004979 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4980 // Frame should not be dropped.
4981 WaitForEncodedFrame(2);
4982
Henrik Boström381d1092020-05-12 18:49:07 +02004983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004984 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4985 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4986 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004987 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4988 // Expect to drop this frame, the wait should time out.
4989 ExpectDroppedFrame();
4990
4991 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004992 EXPECT_TRUE_WAIT(
4993 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004994 video_stream_encoder_->Stop();
4995}
4996
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004997TEST_F(VideoStreamEncoderTest,
4998 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
4999 webrtc::test::ScopedFieldTrials field_trials(
5000 "WebRTC-Video-QualityScalerSettings/"
5001 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5002 fake_encoder_.SetQualityScaling(false);
5003 ConfigureEncoder(video_encoder_config_.Copy());
5004 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5005 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5006 const int kWidth = 640;
5007 const int kHeight = 360;
5008
5009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5010 DataRate::BitsPerSec(kTargetBitrateBps),
5011 DataRate::BitsPerSec(kTargetBitrateBps),
5012 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5013 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5014 // Frame should not be dropped.
5015 WaitForEncodedFrame(1);
5016
5017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5018 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5019 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5020 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5021 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5022 // Frame should not be dropped.
5023 WaitForEncodedFrame(2);
5024
5025 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5026 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5027 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5028 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5029 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5030 // Not dropped since quality scaling is disabled.
5031 WaitForEncodedFrame(3);
5032
5033 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005034 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005035 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5036
5037 video_stream_encoder_->Stop();
5038}
5039
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005040TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5041 const int kLowTargetBitrateBps = 400000;
5042 // Set simulcast.
5043 ResetEncoder("VP8", 3, 1, 1, false);
5044 fake_encoder_.SetQualityScaling(true);
5045 const int kWidth = 1280;
5046 const int kHeight = 720;
5047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5048 DataRate::BitsPerSec(kLowTargetBitrateBps),
5049 DataRate::BitsPerSec(kLowTargetBitrateBps),
5050 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5051 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5052 // Frame should not be dropped.
5053 WaitForEncodedFrame(1);
5054
5055 // Trigger QVGA "singlecast"
5056 // Update the config.
5057 VideoEncoderConfig video_encoder_config;
5058 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5059 &video_encoder_config);
5060 for (auto& layer : video_encoder_config.simulcast_layers) {
5061 layer.num_temporal_layers = 1;
5062 layer.max_framerate = kDefaultFramerate;
5063 }
5064 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5065 video_encoder_config.content_type =
5066 VideoEncoderConfig::ContentType::kRealtimeVideo;
5067
5068 video_encoder_config.simulcast_layers[0].active = true;
5069 video_encoder_config.simulcast_layers[1].active = false;
5070 video_encoder_config.simulcast_layers[2].active = false;
5071
5072 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5073 kMaxPayloadLength);
5074 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5075
5076 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5077 // Frame should not be dropped.
5078 WaitForEncodedFrame(2);
5079
5080 // Trigger HD "singlecast"
5081 video_encoder_config.simulcast_layers[0].active = false;
5082 video_encoder_config.simulcast_layers[1].active = false;
5083 video_encoder_config.simulcast_layers[2].active = true;
5084
5085 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5086 kMaxPayloadLength);
5087 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5088
5089 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5090 // Frame should be dropped because of initial frame drop.
5091 ExpectDroppedFrame();
5092
5093 // Expect the sink_wants to specify a scaled frame.
5094 EXPECT_TRUE_WAIT(
5095 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5096 video_stream_encoder_->Stop();
5097}
5098
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005099TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5100 const int kLowTargetBitrateBps = 400000;
5101 // Set simulcast.
5102 ResetEncoder("VP9", 1, 1, 3, false);
5103 fake_encoder_.SetQualityScaling(true);
5104 const int kWidth = 1280;
5105 const int kHeight = 720;
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5107 DataRate::BitsPerSec(kLowTargetBitrateBps),
5108 DataRate::BitsPerSec(kLowTargetBitrateBps),
5109 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5110 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5111 // Frame should not be dropped.
5112 WaitForEncodedFrame(1);
5113
5114 // Trigger QVGA "singlecast"
5115 // Update the config.
5116 VideoEncoderConfig video_encoder_config;
5117 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5118 &video_encoder_config);
5119 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5120 vp9_settings.numberOfSpatialLayers = 3;
5121 // Since only one layer is active - automatic resize should be enabled.
5122 vp9_settings.automaticResizeOn = true;
5123 video_encoder_config.encoder_specific_settings =
5124 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5125 vp9_settings);
5126 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5127 video_encoder_config.content_type =
5128 VideoEncoderConfig::ContentType::kRealtimeVideo;
5129 // Currently simulcast layers |active| flags are used to inidicate
5130 // which SVC layers are active.
5131 video_encoder_config.simulcast_layers.resize(3);
5132
5133 video_encoder_config.simulcast_layers[0].active = true;
5134 video_encoder_config.simulcast_layers[1].active = false;
5135 video_encoder_config.simulcast_layers[2].active = false;
5136
5137 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5138 kMaxPayloadLength);
5139 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5140
5141 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5142 // Frame should not be dropped.
5143 WaitForEncodedFrame(2);
5144
5145 // Trigger HD "singlecast"
5146 video_encoder_config.simulcast_layers[0].active = false;
5147 video_encoder_config.simulcast_layers[1].active = false;
5148 video_encoder_config.simulcast_layers[2].active = true;
5149
5150 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5151 kMaxPayloadLength);
5152 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5153
5154 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5155 // Frame should be dropped because of initial frame drop.
5156 ExpectDroppedFrame();
5157
5158 // Expect the sink_wants to specify a scaled frame.
5159 EXPECT_TRUE_WAIT(
5160 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5161 video_stream_encoder_->Stop();
5162}
5163
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005164TEST_F(VideoStreamEncoderTest,
5165 InitialFrameDropActivatesWhenResolutionIncreases) {
5166 const int kWidth = 640;
5167 const int kHeight = 360;
5168
5169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5170 DataRate::BitsPerSec(kTargetBitrateBps),
5171 DataRate::BitsPerSec(kTargetBitrateBps),
5172 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5173 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5174 // Frame should not be dropped.
5175 WaitForEncodedFrame(1);
5176
5177 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5178 DataRate::BitsPerSec(kLowTargetBitrateBps),
5179 DataRate::BitsPerSec(kLowTargetBitrateBps),
5180 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5181 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5182 // Frame should not be dropped, bitrate not too low for frame.
5183 WaitForEncodedFrame(2);
5184
5185 // Incoming resolution increases.
5186 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5187 // Expect to drop this frame, bitrate too low for frame.
5188 ExpectDroppedFrame();
5189
5190 // Expect the sink_wants to specify a scaled frame.
5191 EXPECT_TRUE_WAIT(
5192 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5193 video_stream_encoder_->Stop();
5194}
5195
5196TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5197 const int kWidth = 640;
5198 const int kHeight = 360;
5199 // So that quality scaling doesn't happen by itself.
5200 fake_encoder_.SetQp(kQpHigh);
5201
5202 AdaptingFrameForwarder source(&time_controller_);
5203 source.set_adaptation_enabled(true);
5204 video_stream_encoder_->SetSource(
5205 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5206
5207 int timestamp = 1;
5208
5209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5210 DataRate::BitsPerSec(kTargetBitrateBps),
5211 DataRate::BitsPerSec(kTargetBitrateBps),
5212 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5213 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5214 WaitForEncodedFrame(timestamp);
5215 timestamp += 9000;
5216 // Long pause to disable all first BWE drop logic.
5217 AdvanceTime(TimeDelta::Millis(1000));
5218
5219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5220 DataRate::BitsPerSec(kLowTargetBitrateBps),
5221 DataRate::BitsPerSec(kLowTargetBitrateBps),
5222 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5223 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5224 // Not dropped frame, as initial frame drop is disabled by now.
5225 WaitForEncodedFrame(timestamp);
5226 timestamp += 9000;
5227 AdvanceTime(TimeDelta::Millis(100));
5228
5229 // Quality adaptation down.
5230 video_stream_encoder_->TriggerQualityLow();
5231
5232 // Adaptation has an effect.
5233 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5234 5000);
5235
5236 // Frame isn't dropped as initial frame dropper is disabled.
5237 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5238 WaitForEncodedFrame(timestamp);
5239 timestamp += 9000;
5240 AdvanceTime(TimeDelta::Millis(100));
5241
5242 // Quality adaptation up.
5243 video_stream_encoder_->TriggerQualityHigh();
5244
5245 // Adaptation has an effect.
5246 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5247 5000);
5248
5249 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5250 // Frame should not be dropped, as initial framedropper is off.
5251 WaitForEncodedFrame(timestamp);
5252
5253 video_stream_encoder_->Stop();
5254}
5255
Åsa Perssone644a032019-11-08 15:56:00 +01005256TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5257 webrtc::test::ScopedFieldTrials field_trials(
5258 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5259
5260 // Reset encoder for field trials to take effect.
5261 VideoEncoderConfig config = video_encoder_config_.Copy();
5262 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005263 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005264 ConfigureEncoder(std::move(config));
5265 fake_encoder_.SetQp(kQpLow);
5266
5267 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005268 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005269 source.set_adaptation_enabled(true);
5270 video_stream_encoder_->SetSource(&source,
5271 DegradationPreference::MAINTAIN_FRAMERATE);
5272
5273 // Start at low bitrate.
5274 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005275 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5276 DataRate::BitsPerSec(kLowBitrateBps),
5277 DataRate::BitsPerSec(kLowBitrateBps),
5278 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005279
5280 // Expect first frame to be dropped and resolution to be limited.
5281 const int kWidth = 1280;
5282 const int kHeight = 720;
5283 const int64_t kFrameIntervalMs = 100;
5284 int64_t timestamp_ms = kFrameIntervalMs;
5285 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5286 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005287 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5288 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005289
5290 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5292 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005293
5294 // Insert frames and advance |min_duration_ms|.
5295 for (size_t i = 1; i <= 10; i++) {
5296 timestamp_ms += kFrameIntervalMs;
5297 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5298 WaitForEncodedFrame(timestamp_ms);
5299 }
5300 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5301 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5302
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005303 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005304
5305 // Insert frame should trigger high BW and release quality limitation.
5306 timestamp_ms += kFrameIntervalMs;
5307 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5308 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005309 // The ramp-up code involves the adaptation queue, give it time to execute.
5310 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005311 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005312 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005313
5314 // Frame should not be adapted.
5315 timestamp_ms += kFrameIntervalMs;
5316 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5317 WaitForEncodedFrame(kWidth, kHeight);
5318 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5319
5320 video_stream_encoder_->Stop();
5321}
5322
mflodmancc3d4422017-08-03 08:27:51 -07005323TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005324 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005325 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005326 source.set_adaptation_enabled(true);
5327 video_stream_encoder_->SetSource(&source,
5328 DegradationPreference::MAINTAIN_FRAMERATE);
5329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5330 DataRate::BitsPerSec(kTargetBitrateBps),
5331 DataRate::BitsPerSec(kTargetBitrateBps),
5332 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5333 fake_encoder_.SetQp(kQpHigh + 1);
5334 const int kWidth = 1280;
5335 const int kHeight = 720;
5336 const int64_t kFrameIntervalMs = 100;
5337 int64_t timestamp_ms = kFrameIntervalMs;
5338 for (size_t i = 1; i <= 100; i++) {
5339 timestamp_ms += kFrameIntervalMs;
5340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5341 WaitForEncodedFrame(timestamp_ms);
5342 }
5343 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5344 // for the first time.
5345 // TODO(eshr): We should avoid these waits by using threads with simulated
5346 // time.
5347 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5348 2000 * 2.5 * 2);
5349 timestamp_ms += kFrameIntervalMs;
5350 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5351 WaitForEncodedFrame(timestamp_ms);
5352 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5353 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5354 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5355
5356 // Disable Quality scaling by turning off scaler on the encoder and
5357 // reconfiguring.
5358 fake_encoder_.SetQualityScaling(false);
5359 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5360 kMaxPayloadLength);
5361 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005362 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005363 // Since we turned off the quality scaler, the adaptations made by it are
5364 // removed.
5365 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5366 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5367
5368 video_stream_encoder_->Stop();
5369}
5370
5371TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005372 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5373 const int kTooSmallWidth = 10;
5374 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005375 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005376 DataRate::BitsPerSec(kTargetBitrateBps),
5377 DataRate::BitsPerSec(kTargetBitrateBps),
5378 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005379
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005380 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005381 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005382 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005383 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005384 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005385 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5386
5387 // Trigger adapt down, too small frame, expect no change.
5388 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005389 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005390 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005391 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005392 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5393 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5394
mflodmancc3d4422017-08-03 08:27:51 -07005395 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005396}
5397
mflodmancc3d4422017-08-03 08:27:51 -07005398TEST_F(VideoStreamEncoderTest,
5399 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005400 const int kTooSmallWidth = 10;
5401 const int kTooSmallHeight = 10;
5402 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005403 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005404 DataRate::BitsPerSec(kTargetBitrateBps),
5405 DataRate::BitsPerSec(kTargetBitrateBps),
5406 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005407
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005408 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07005409 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005410 video_stream_encoder_->SetSource(&source,
5411 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005412 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07005413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5415
5416 // Trigger adapt down, expect limited framerate.
5417 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005418 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005419 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005420 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5424
5425 // Trigger adapt down, too small frame, expect no change.
5426 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005427 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07005428 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005429 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005430 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5431 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5432 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5433
mflodmancc3d4422017-08-03 08:27:51 -07005434 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005435}
5436
mflodmancc3d4422017-08-03 08:27:51 -07005437TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07005438 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02005439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005440 DataRate::BitsPerSec(kTargetBitrateBps),
5441 DataRate::BitsPerSec(kTargetBitrateBps),
5442 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02005443 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07005444 const int kFrameWidth = 1280;
5445 const int kFrameHeight = 720;
5446 video_source_.IncomingCapturedFrame(
5447 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005448 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07005449 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07005450}
5451
sprangb1ca0732017-02-01 08:38:12 -08005452// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07005453TEST_F(VideoStreamEncoderTest,
5454 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02005455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005456 DataRate::BitsPerSec(kTargetBitrateBps),
5457 DataRate::BitsPerSec(kTargetBitrateBps),
5458 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08005459
5460 const int kFrameWidth = 1280;
5461 const int kFrameHeight = 720;
5462 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07005463 // requested by
5464 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08005465 video_source_.set_adaptation_enabled(true);
5466
5467 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005468 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005469 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005470
5471 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005472 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08005473 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005474 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005475 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08005476
asaperssonfab67072017-04-04 05:51:49 -07005477 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02005478 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08005479 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005480 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005481 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005482
mflodmancc3d4422017-08-03 08:27:51 -07005483 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08005484}
sprangfe627f32017-03-29 08:24:59 -07005485
mflodmancc3d4422017-08-03 08:27:51 -07005486TEST_F(VideoStreamEncoderTest,
5487 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07005488 const int kFrameWidth = 1280;
5489 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07005490
Henrik Boström381d1092020-05-12 18:49:07 +02005491 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005492 DataRate::BitsPerSec(kTargetBitrateBps),
5493 DataRate::BitsPerSec(kTargetBitrateBps),
5494 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005495 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005496 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005497 video_source_.set_adaptation_enabled(true);
5498
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005499 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005500
5501 video_source_.IncomingCapturedFrame(
5502 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005503 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005504
5505 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07005506 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005507
5508 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07005509 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005510 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005511 video_source_.IncomingCapturedFrame(
5512 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005513 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005514 }
5515
5516 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07005517 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005518 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005519 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005520 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005521 video_source_.IncomingCapturedFrame(
5522 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005523 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005524 ++num_frames_dropped;
5525 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005526 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005527 }
5528 }
5529
sprang4847ae62017-06-27 07:06:52 -07005530 // Add some slack to account for frames dropped by the frame dropper.
5531 const int kErrorMargin = 1;
5532 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005533 kErrorMargin);
5534
5535 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07005536 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005537 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02005538 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005539 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005540 video_source_.IncomingCapturedFrame(
5541 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005542 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005543 ++num_frames_dropped;
5544 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005545 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005546 }
5547 }
sprang4847ae62017-06-27 07:06:52 -07005548 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07005549 kErrorMargin);
5550
5551 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02005552 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005553 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005554 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005555 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005556 video_source_.IncomingCapturedFrame(
5557 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005558 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005559 ++num_frames_dropped;
5560 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005561 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005562 }
5563 }
sprang4847ae62017-06-27 07:06:52 -07005564 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005565 kErrorMargin);
5566
5567 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02005568 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005569 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005570 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005571 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005572 video_source_.IncomingCapturedFrame(
5573 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005574 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005575 ++num_frames_dropped;
5576 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005577 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005578 }
5579 }
5580 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
5581
mflodmancc3d4422017-08-03 08:27:51 -07005582 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005583}
5584
mflodmancc3d4422017-08-03 08:27:51 -07005585TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07005586 const int kFramerateFps = 5;
5587 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07005588 const int kFrameWidth = 1280;
5589 const int kFrameHeight = 720;
5590
sprang4847ae62017-06-27 07:06:52 -07005591 // Reconfigure encoder with two temporal layers and screensharing, which will
5592 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02005593 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07005594
Henrik Boström381d1092020-05-12 18:49:07 +02005595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005596 DataRate::BitsPerSec(kTargetBitrateBps),
5597 DataRate::BitsPerSec(kTargetBitrateBps),
5598 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005599 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005600 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005601 video_source_.set_adaptation_enabled(true);
5602
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005603 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005604
5605 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08005606 rtc::VideoSinkWants last_wants;
5607 do {
5608 last_wants = video_source_.sink_wants();
5609
sprangc5d62e22017-04-02 23:53:04 -07005610 // Insert frames to get a new fps estimate...
5611 for (int j = 0; j < kFramerateFps; ++j) {
5612 video_source_.IncomingCapturedFrame(
5613 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08005614 if (video_source_.last_sent_width()) {
5615 sink_.WaitForEncodedFrame(timestamp_ms);
5616 }
sprangc5d62e22017-04-02 23:53:04 -07005617 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005618 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07005619 }
5620 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07005621 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08005622 } while (video_source_.sink_wants().max_framerate_fps <
5623 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07005624
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005625 EXPECT_THAT(video_source_.sink_wants(),
5626 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07005627
mflodmancc3d4422017-08-03 08:27:51 -07005628 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005629}
asaperssonf7e294d2017-06-13 23:25:22 -07005630
mflodmancc3d4422017-08-03 08:27:51 -07005631TEST_F(VideoStreamEncoderTest,
5632 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005633 const int kWidth = 1280;
5634 const int kHeight = 720;
5635 const int64_t kFrameIntervalMs = 150;
5636 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005638 DataRate::BitsPerSec(kTargetBitrateBps),
5639 DataRate::BitsPerSec(kTargetBitrateBps),
5640 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005641
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005642 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005643 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005644 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005645 video_stream_encoder_->SetSource(&source,
5646 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005647 timestamp_ms += kFrameIntervalMs;
5648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005649 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005650 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5652 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5654
5655 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005656 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005657 timestamp_ms += kFrameIntervalMs;
5658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005659 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005660 EXPECT_THAT(source.sink_wants(),
5661 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005662 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5664 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5665
5666 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005667 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005668 timestamp_ms += kFrameIntervalMs;
5669 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005670 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005671 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005672 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5674 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5675
5676 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005677 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005678 timestamp_ms += kFrameIntervalMs;
5679 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005680 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005681 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005682 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5683 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5684 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5685
5686 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005687 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005688 timestamp_ms += kFrameIntervalMs;
5689 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005690 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005691 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005692 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5693 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5694 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5695
5696 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005697 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005698 timestamp_ms += kFrameIntervalMs;
5699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005700 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005701 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005702 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5703 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5704 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5705
5706 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005707 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005708 timestamp_ms += kFrameIntervalMs;
5709 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005710 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005711 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005712 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5713 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5714 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5715
5716 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07005717 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005718 timestamp_ms += kFrameIntervalMs;
5719 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005720 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005721 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005722 rtc::VideoSinkWants last_wants = source.sink_wants();
5723 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5724 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5725 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5726
5727 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005728 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005729 timestamp_ms += kFrameIntervalMs;
5730 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005731 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005732 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07005733 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5734 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5735 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5736
Evan Shrubsole64469032020-06-11 10:45:29 +02005737 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005738 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005739 timestamp_ms += kFrameIntervalMs;
5740 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005741 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005742 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005743 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5744 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5745 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5746
5747 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005748 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005749 timestamp_ms += kFrameIntervalMs;
5750 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005751 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005752 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005753 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5754 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5755 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5756
5757 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005758 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005759 timestamp_ms += kFrameIntervalMs;
5760 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005761 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005762 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005763 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5764 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5765 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5766
5767 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005768 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005769 timestamp_ms += kFrameIntervalMs;
5770 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005771 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005772 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005773 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5774 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5775 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5776
5777 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005778 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005779 timestamp_ms += kFrameIntervalMs;
5780 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005781 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005782 EXPECT_THAT(source.sink_wants(), FpsMax());
5783 EXPECT_EQ(source.sink_wants().max_pixel_count,
5784 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07005785 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5786 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5787 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5788
5789 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005790 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005791 timestamp_ms += kFrameIntervalMs;
5792 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005793 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005794 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005795 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5797 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5798
Åsa Persson30ab0152019-08-27 12:22:33 +02005799 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005800 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005801 timestamp_ms += kFrameIntervalMs;
5802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005803 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5808 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5809
5810 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005811 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005812 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005813 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5814
mflodmancc3d4422017-08-03 08:27:51 -07005815 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005816}
5817
mflodmancc3d4422017-08-03 08:27:51 -07005818TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07005819 const int kWidth = 1280;
5820 const int kHeight = 720;
5821 const int64_t kFrameIntervalMs = 150;
5822 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005824 DataRate::BitsPerSec(kTargetBitrateBps),
5825 DataRate::BitsPerSec(kTargetBitrateBps),
5826 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005827
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005828 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005829 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005830 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005831 video_stream_encoder_->SetSource(&source,
5832 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005833 timestamp_ms += kFrameIntervalMs;
5834 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005835 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005836 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005837 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5838 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5839 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5840 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5841 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5842 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5843
5844 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005845 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005846 timestamp_ms += kFrameIntervalMs;
5847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005848 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005849 EXPECT_THAT(source.sink_wants(),
5850 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5853 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5855 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5857
5858 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005859 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005860 timestamp_ms += kFrameIntervalMs;
5861 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005862 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005863 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5866 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5868 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5869 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5870
5871 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005872 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005873 timestamp_ms += kFrameIntervalMs;
5874 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005875 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005876 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02005877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07005878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5879 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5880 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5881 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5882 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5883
Evan Shrubsole64469032020-06-11 10:45:29 +02005884 // Trigger cpu adapt up, expect no change since QP is most limited.
5885 {
5886 // Store current sink wants since we expect no change and if there is no
5887 // change then last_wants() is not updated.
5888 auto previous_sink_wants = source.sink_wants();
5889 video_stream_encoder_->TriggerCpuUnderuse();
5890 timestamp_ms += kFrameIntervalMs;
5891 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5892 WaitForEncodedFrame(timestamp_ms);
5893 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
5894 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5896 }
5897
5898 // Trigger quality adapt up, expect increased fps (640x360@30fps).
5899 video_stream_encoder_->TriggerQualityHigh();
5900 timestamp_ms += kFrameIntervalMs;
5901 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5902 WaitForEncodedFrame(timestamp_ms);
5903 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
5904 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5905 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5906 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5907 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5908 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5909 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5910
5911 // Trigger quality adapt up and Cpu adapt up since both are most limited,
5912 // expect increased resolution (960x540@30fps).
5913 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02005914 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005915 timestamp_ms += kFrameIntervalMs;
5916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005917 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02005918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005919 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5921 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
5922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5923 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005924 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005925
Evan Shrubsole64469032020-06-11 10:45:29 +02005926 // Trigger quality adapt up and Cpu adapt up since both are most limited,
5927 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005928 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02005929 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005930 timestamp_ms += kFrameIntervalMs;
5931 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005932 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005933 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005934 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005935 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5936 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5937 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5938 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5939 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005940 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005941
5942 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005943 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005944 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005945 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005946 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005947
mflodmancc3d4422017-08-03 08:27:51 -07005948 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005949}
5950
mflodmancc3d4422017-08-03 08:27:51 -07005951TEST_F(VideoStreamEncoderTest,
5952 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07005953 const int kWidth = 640;
5954 const int kHeight = 360;
5955 const int kFpsLimit = 15;
5956 const int64_t kFrameIntervalMs = 150;
5957 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005959 DataRate::BitsPerSec(kTargetBitrateBps),
5960 DataRate::BitsPerSec(kTargetBitrateBps),
5961 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005962
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005963 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005964 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005965 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005966 video_stream_encoder_->SetSource(&source,
5967 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005968 timestamp_ms += kFrameIntervalMs;
5969 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005970 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005971 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005972 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5973 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5974 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5975 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5976 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5977 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5978
5979 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005980 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005981 timestamp_ms += kFrameIntervalMs;
5982 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005983 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005984 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005985 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5986 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5987 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5988 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5989 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5990 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5991
5992 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005993 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005994 timestamp_ms += kFrameIntervalMs;
5995 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005996 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005997 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005998 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02005999 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006000 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6001 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6002 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6003 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6004
Evan Shrubsole64469032020-06-11 10:45:29 +02006005 // Trigger cpu adapt up, expect no change because quality is most limited.
6006 {
6007 auto previous_sink_wants = source.sink_wants();
6008 // Store current sink wants since we expect no change ind if there is no
6009 // change then last__wants() is not updated.
6010 video_stream_encoder_->TriggerCpuUnderuse();
6011 timestamp_ms += kFrameIntervalMs;
6012 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6013 WaitForEncodedFrame(timestamp_ms);
6014 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6015 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6016 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6017 }
6018
6019 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6020 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006021 timestamp_ms += kFrameIntervalMs;
6022 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006023 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006024 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006025 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6026 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6027 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006028 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6029 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6030 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006031
Evan Shrubsole64469032020-06-11 10:45:29 +02006032 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006033 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006034 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006035 timestamp_ms += kFrameIntervalMs;
6036 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006037 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006038 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006039 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6040 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6041 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6042 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6043 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006044 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006045
6046 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006047 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006048 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006049 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006050 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006051
mflodmancc3d4422017-08-03 08:27:51 -07006052 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006053}
6054
mflodmancc3d4422017-08-03 08:27:51 -07006055TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006056 const int kFrameWidth = 1920;
6057 const int kFrameHeight = 1080;
6058 // 3/4 of 1920.
6059 const int kAdaptedFrameWidth = 1440;
6060 // 3/4 of 1080 rounded down to multiple of 4.
6061 const int kAdaptedFrameHeight = 808;
6062 const int kFramerate = 24;
6063
Henrik Boström381d1092020-05-12 18:49:07 +02006064 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006065 DataRate::BitsPerSec(kTargetBitrateBps),
6066 DataRate::BitsPerSec(kTargetBitrateBps),
6067 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006068 // Trigger reconfigure encoder (without resetting the entire instance).
6069 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006070 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6071 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006072 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006073 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006074 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006075 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006076 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006077 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006078
6079 video_source_.set_adaptation_enabled(true);
6080
6081 video_source_.IncomingCapturedFrame(
6082 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006083 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006084
6085 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006086 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006087 video_source_.IncomingCapturedFrame(
6088 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006089 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006090
mflodmancc3d4422017-08-03 08:27:51 -07006091 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006092}
6093
mflodmancc3d4422017-08-03 08:27:51 -07006094TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006095 const int kFrameWidth = 1280;
6096 const int kFrameHeight = 720;
6097 const int kLowFps = 2;
6098 const int kHighFps = 30;
6099
Henrik Boström381d1092020-05-12 18:49:07 +02006100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006101 DataRate::BitsPerSec(kTargetBitrateBps),
6102 DataRate::BitsPerSec(kTargetBitrateBps),
6103 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006104
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006105 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006106 max_framerate_ = kLowFps;
6107
6108 // Insert 2 seconds of 2fps video.
6109 for (int i = 0; i < kLowFps * 2; ++i) {
6110 video_source_.IncomingCapturedFrame(
6111 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6112 WaitForEncodedFrame(timestamp_ms);
6113 timestamp_ms += 1000 / kLowFps;
6114 }
6115
6116 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006118 DataRate::BitsPerSec(kTargetBitrateBps),
6119 DataRate::BitsPerSec(kTargetBitrateBps),
6120 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006121 video_source_.IncomingCapturedFrame(
6122 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6123 WaitForEncodedFrame(timestamp_ms);
6124 timestamp_ms += 1000 / kLowFps;
6125
6126 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6127
6128 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006129 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006130 const int kFrameIntervalMs = 1000 / kHighFps;
6131 max_framerate_ = kHighFps;
6132 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6133 video_source_.IncomingCapturedFrame(
6134 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6135 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6136 // be dropped if the encoder hans't been updated with the new higher target
6137 // framerate yet, causing it to overshoot the target bitrate and then
6138 // suffering the wrath of the media optimizer.
6139 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6140 timestamp_ms += kFrameIntervalMs;
6141 }
6142
6143 // Don expect correct measurement just yet, but it should be higher than
6144 // before.
6145 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6146
mflodmancc3d4422017-08-03 08:27:51 -07006147 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006148}
6149
mflodmancc3d4422017-08-03 08:27:51 -07006150TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006151 const int kFrameWidth = 1280;
6152 const int kFrameHeight = 720;
6153 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006154 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006155 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006156 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006157
Henrik Boström381d1092020-05-12 18:49:07 +02006158 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006159 DataRate::BitsPerSec(kTargetBitrateBps),
6160 DataRate::BitsPerSec(kTargetBitrateBps),
6161 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006162 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006163
6164 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006165 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006166 video_source_.IncomingCapturedFrame(
6167 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6168 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006169 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006170
6171 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006172 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6173 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6174 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006175
6176 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006177 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006178 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006179
Per Kjellanderdcef6412020-10-07 15:09:05 +02006180 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006181 video_source_.IncomingCapturedFrame(
6182 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6183 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006184 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006185
mflodmancc3d4422017-08-03 08:27:51 -07006186 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006187}
ilnik6b826ef2017-06-16 06:53:48 -07006188
Niels Möller4db138e2018-04-19 09:04:13 +02006189TEST_F(VideoStreamEncoderTest,
6190 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6191 const int kFrameWidth = 1280;
6192 const int kFrameHeight = 720;
6193 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006194 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006195 DataRate::BitsPerSec(kTargetBitrateBps),
6196 DataRate::BitsPerSec(kTargetBitrateBps),
6197 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006198 video_source_.IncomingCapturedFrame(
6199 CreateFrame(1, kFrameWidth, kFrameHeight));
6200 WaitForEncodedFrame(1);
6201 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6202 .low_encode_usage_threshold_percent,
6203 default_options.low_encode_usage_threshold_percent);
6204 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6205 .high_encode_usage_threshold_percent,
6206 default_options.high_encode_usage_threshold_percent);
6207 video_stream_encoder_->Stop();
6208}
6209
6210TEST_F(VideoStreamEncoderTest,
6211 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6212 const int kFrameWidth = 1280;
6213 const int kFrameHeight = 720;
6214 CpuOveruseOptions hardware_options;
6215 hardware_options.low_encode_usage_threshold_percent = 150;
6216 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006217 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006218
Henrik Boström381d1092020-05-12 18:49:07 +02006219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006220 DataRate::BitsPerSec(kTargetBitrateBps),
6221 DataRate::BitsPerSec(kTargetBitrateBps),
6222 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006223 video_source_.IncomingCapturedFrame(
6224 CreateFrame(1, kFrameWidth, kFrameHeight));
6225 WaitForEncodedFrame(1);
6226 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6227 .low_encode_usage_threshold_percent,
6228 hardware_options.low_encode_usage_threshold_percent);
6229 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6230 .high_encode_usage_threshold_percent,
6231 hardware_options.high_encode_usage_threshold_percent);
6232 video_stream_encoder_->Stop();
6233}
6234
Niels Möller6bb5ab92019-01-11 11:11:10 +01006235TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6236 const int kFrameWidth = 320;
6237 const int kFrameHeight = 240;
6238 const int kFps = 30;
6239 const int kTargetBitrateBps = 120000;
6240 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6241
Henrik Boström381d1092020-05-12 18:49:07 +02006242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006243 DataRate::BitsPerSec(kTargetBitrateBps),
6244 DataRate::BitsPerSec(kTargetBitrateBps),
6245 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006246
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006247 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006248 max_framerate_ = kFps;
6249
6250 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6251 fake_encoder_.SimulateOvershoot(1.0);
6252 int num_dropped = 0;
6253 for (int i = 0; i < kNumFramesInRun; ++i) {
6254 video_source_.IncomingCapturedFrame(
6255 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6256 // Wait up to two frame durations for a frame to arrive.
6257 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6258 ++num_dropped;
6259 }
6260 timestamp_ms += 1000 / kFps;
6261 }
6262
Erik Språnga8d48ab2019-02-08 14:17:40 +01006263 // Framerate should be measured to be near the expected target rate.
6264 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6265
6266 // Frame drops should be within 5% of expected 0%.
6267 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006268
6269 // Make encoder produce frames at double the expected bitrate during 3 seconds
6270 // of video, verify number of drops. Rate needs to be slightly changed in
6271 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006272 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006273 const RateControlSettings trials =
6274 RateControlSettings::ParseFromFieldTrials();
6275 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006276 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006277 // frame dropping since the adjuter will try to just lower the target
6278 // bitrate rather than drop frames. If network headroom can be used, it
6279 // doesn't push back as hard so we don't need quite as much overshoot.
6280 // These numbers are unfortunately a bit magical but there's not trivial
6281 // way to algebraically infer them.
6282 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6283 overshoot_factor = 2.4;
6284 } else {
6285 overshoot_factor = 4.0;
6286 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006287 }
6288 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006289 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006290 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6291 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6292 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006293 num_dropped = 0;
6294 for (int i = 0; i < kNumFramesInRun; ++i) {
6295 video_source_.IncomingCapturedFrame(
6296 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6297 // Wait up to two frame durations for a frame to arrive.
6298 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6299 ++num_dropped;
6300 }
6301 timestamp_ms += 1000 / kFps;
6302 }
6303
Henrik Boström381d1092020-05-12 18:49:07 +02006304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006305 DataRate::BitsPerSec(kTargetBitrateBps),
6306 DataRate::BitsPerSec(kTargetBitrateBps),
6307 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006308
6309 // Target framerate should be still be near the expected target, despite
6310 // the frame drops.
6311 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6312
6313 // Frame drops should be within 5% of expected 50%.
6314 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006315
6316 video_stream_encoder_->Stop();
6317}
6318
6319TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6320 const int kFrameWidth = 320;
6321 const int kFrameHeight = 240;
6322 const int kActualInputFps = 24;
6323 const int kTargetBitrateBps = 120000;
6324
6325 ASSERT_GT(max_framerate_, kActualInputFps);
6326
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006327 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006328 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006330 DataRate::BitsPerSec(kTargetBitrateBps),
6331 DataRate::BitsPerSec(kTargetBitrateBps),
6332 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006333
6334 // Insert 3 seconds of video, with an input fps lower than configured max.
6335 for (int i = 0; i < kActualInputFps * 3; ++i) {
6336 video_source_.IncomingCapturedFrame(
6337 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6338 // Wait up to two frame durations for a frame to arrive.
6339 WaitForEncodedFrame(timestamp_ms);
6340 timestamp_ms += 1000 / kActualInputFps;
6341 }
6342
6343 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6344
6345 video_stream_encoder_->Stop();
6346}
6347
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006348TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006349 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006350 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006351 DataRate::BitsPerSec(kTargetBitrateBps),
6352 DataRate::BitsPerSec(kTargetBitrateBps),
6353 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006354
6355 fake_encoder_.BlockNextEncode();
6356 video_source_.IncomingCapturedFrame(
6357 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6358 WaitForEncodedFrame(1);
6359 // On the very first frame full update should be forced.
6360 rect = fake_encoder_.GetLastUpdateRect();
6361 EXPECT_EQ(rect.offset_x, 0);
6362 EXPECT_EQ(rect.offset_y, 0);
6363 EXPECT_EQ(rect.height, codec_height_);
6364 EXPECT_EQ(rect.width, codec_width_);
6365 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6366 // call to ContinueEncode.
6367 video_source_.IncomingCapturedFrame(
6368 CreateFrameWithUpdatedPixel(2, nullptr, 1));
6369 ExpectDroppedFrame();
6370 video_source_.IncomingCapturedFrame(
6371 CreateFrameWithUpdatedPixel(3, nullptr, 10));
6372 ExpectDroppedFrame();
6373 fake_encoder_.ContinueEncode();
6374 WaitForEncodedFrame(3);
6375 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
6376 rect = fake_encoder_.GetLastUpdateRect();
6377 EXPECT_EQ(rect.offset_x, 1);
6378 EXPECT_EQ(rect.offset_y, 0);
6379 EXPECT_EQ(rect.width, 10);
6380 EXPECT_EQ(rect.height, 1);
6381
6382 video_source_.IncomingCapturedFrame(
6383 CreateFrameWithUpdatedPixel(4, nullptr, 0));
6384 WaitForEncodedFrame(4);
6385 // Previous frame was encoded, so no accumulation should happen.
6386 rect = fake_encoder_.GetLastUpdateRect();
6387 EXPECT_EQ(rect.offset_x, 0);
6388 EXPECT_EQ(rect.offset_y, 0);
6389 EXPECT_EQ(rect.width, 1);
6390 EXPECT_EQ(rect.height, 1);
6391
6392 video_stream_encoder_->Stop();
6393}
6394
Erik Språngd7329ca2019-02-21 21:19:53 +01006395TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02006396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006397 DataRate::BitsPerSec(kTargetBitrateBps),
6398 DataRate::BitsPerSec(kTargetBitrateBps),
6399 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006400
6401 // First frame is always keyframe.
6402 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6403 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01006404 EXPECT_THAT(
6405 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006406 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006407
6408 // Insert delta frame.
6409 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6410 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01006411 EXPECT_THAT(
6412 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006413 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006414
6415 // Request next frame be a key-frame.
6416 video_stream_encoder_->SendKeyFrame();
6417 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6418 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01006419 EXPECT_THAT(
6420 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006421 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006422
6423 video_stream_encoder_->Stop();
6424}
6425
6426TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
6427 // Setup simulcast with three streams.
6428 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006429 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006430 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6431 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6432 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006433 // Wait for all three layers before triggering event.
6434 sink_.SetNumExpectedLayers(3);
6435
6436 // First frame is always keyframe.
6437 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6438 WaitForEncodedFrame(1);
6439 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006440 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6441 VideoFrameType::kVideoFrameKey,
6442 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006443
6444 // Insert delta frame.
6445 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6446 WaitForEncodedFrame(2);
6447 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006448 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6449 VideoFrameType::kVideoFrameDelta,
6450 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006451
6452 // Request next frame be a key-frame.
6453 // Only first stream is configured to produce key-frame.
6454 video_stream_encoder_->SendKeyFrame();
6455 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6456 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02006457
6458 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
6459 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01006460 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006461 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02006462 VideoFrameType::kVideoFrameKey,
6463 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006464
6465 video_stream_encoder_->Stop();
6466}
6467
6468TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
6469 // Configure internal source factory and setup test again.
6470 encoder_factory_.SetHasInternalSource(true);
6471 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006473 DataRate::BitsPerSec(kTargetBitrateBps),
6474 DataRate::BitsPerSec(kTargetBitrateBps),
6475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006476
6477 // Call encoder directly, simulating internal source where encoded frame
6478 // callback in VideoStreamEncoder is called despite no OnFrame().
6479 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
6480 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006481 EXPECT_THAT(
6482 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006483 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006484
Niels Möller8f7ce222019-03-21 15:43:58 +01006485 const std::vector<VideoFrameType> kDeltaFrame = {
6486 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01006487 // Need to set timestamp manually since manually for injected frame.
6488 VideoFrame frame = CreateFrame(101, nullptr);
6489 frame.set_timestamp(101);
6490 fake_encoder_.InjectFrame(frame, false);
6491 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006492 EXPECT_THAT(
6493 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006494 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006495
6496 // Request key-frame. The forces a dummy frame down into the encoder.
6497 fake_encoder_.ExpectNullFrame();
6498 video_stream_encoder_->SendKeyFrame();
6499 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006500 EXPECT_THAT(
6501 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006502 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006503
6504 video_stream_encoder_->Stop();
6505}
Erik Språngb7cb7b52019-02-26 15:52:33 +01006506
6507TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
6508 // Configure internal source factory and setup test again.
6509 encoder_factory_.SetHasInternalSource(true);
6510 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006512 DataRate::BitsPerSec(kTargetBitrateBps),
6513 DataRate::BitsPerSec(kTargetBitrateBps),
6514 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006515
6516 int64_t timestamp = 1;
6517 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02006518 image.SetEncodedData(
6519 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01006520 image.capture_time_ms_ = ++timestamp;
6521 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
6522 const int64_t kEncodeFinishDelayMs = 10;
6523 image.timing_.encode_start_ms = timestamp;
6524 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
6525 fake_encoder_.InjectEncodedImage(image);
6526 // Wait for frame without incrementing clock.
6527 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
6528 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
6529 // capture timestamp should be kEncodeFinishDelayMs in the past.
6530 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006531 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006532
6533 video_stream_encoder_->Stop();
6534}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006535
6536TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006537 // SPS contains VUI with restrictions on the maximum number of reordered
6538 // pictures, there is no need to rewrite the bitstream to enable faster
6539 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006540 ResetEncoder("H264", 1, 1, 1, false);
6541
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6543 DataRate::BitsPerSec(kTargetBitrateBps),
6544 DataRate::BitsPerSec(kTargetBitrateBps),
6545 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6546 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006547
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006548 fake_encoder_.SetEncodedImageData(
6549 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006550
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006551 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6552 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006553
6554 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6555 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006556
6557 video_stream_encoder_->Stop();
6558}
6559
6560TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006561 // SPS does not contain VUI, the bitstream is will be rewritten with added
6562 // VUI with restrictions on the maximum number of reordered pictures to
6563 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006564 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
6565 0x00, 0x00, 0x03, 0x03, 0xF4,
6566 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006567 ResetEncoder("H264", 1, 1, 1, false);
6568
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6570 DataRate::BitsPerSec(kTargetBitrateBps),
6571 DataRate::BitsPerSec(kTargetBitrateBps),
6572 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6573 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006574
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006575 fake_encoder_.SetEncodedImageData(
6576 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006577
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006578 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6579 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006580
6581 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6582 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006583
6584 video_stream_encoder_->Stop();
6585}
6586
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006587TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
6588 const int kFrameWidth = 1280;
6589 const int kFrameHeight = 720;
6590 const int kTargetBitrateBps = 300000; // To low for HD resolution.
6591
Henrik Boström381d1092020-05-12 18:49:07 +02006592 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006593 DataRate::BitsPerSec(kTargetBitrateBps),
6594 DataRate::BitsPerSec(kTargetBitrateBps),
6595 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006596 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6597
6598 // Insert a first video frame. It should be dropped because of downscale in
6599 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006600 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006601 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6602 frame.set_rotation(kVideoRotation_270);
6603 video_source_.IncomingCapturedFrame(frame);
6604
6605 ExpectDroppedFrame();
6606
6607 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006608 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006609 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6610 frame.set_rotation(kVideoRotation_90);
6611 video_source_.IncomingCapturedFrame(frame);
6612
6613 WaitForEncodedFrame(timestamp_ms);
6614 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
6615
6616 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006617 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006618 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6619 frame.set_rotation(kVideoRotation_180);
6620 video_source_.IncomingCapturedFrame(frame);
6621
6622 WaitForEncodedFrame(timestamp_ms);
6623 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
6624
6625 video_stream_encoder_->Stop();
6626}
6627
Erik Språng5056af02019-09-02 15:53:11 +02006628TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
6629 const int kFrameWidth = 320;
6630 const int kFrameHeight = 180;
6631
6632 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02006633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006634 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
6635 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
6636 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02006637 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006638 /*rtt_ms=*/0,
6639 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006640
6641 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006642 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02006643 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6644 frame.set_rotation(kVideoRotation_270);
6645 video_source_.IncomingCapturedFrame(frame);
6646 WaitForEncodedFrame(timestamp_ms);
6647
6648 // Set a target rate below the minimum allowed by the codec settings.
6649 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006650 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
6651 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02006652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02006653 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02006654 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02006655 /*link_allocation=*/target_rate,
6656 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006657 /*rtt_ms=*/0,
6658 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006659 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6660
6661 // Target bitrate and bandwidth allocation should both be capped at min_rate.
6662 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6663 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006664 DataRate allocation_sum =
6665 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02006666 EXPECT_EQ(min_rate, allocation_sum);
6667 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
6668
6669 video_stream_encoder_->Stop();
6670}
6671
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006672TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02006673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006674 DataRate::BitsPerSec(kTargetBitrateBps),
6675 DataRate::BitsPerSec(kTargetBitrateBps),
6676 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006677 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006678 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006679 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6680 WaitForEncodedFrame(1);
6681
6682 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6683 ASSERT_TRUE(prev_rate_settings.has_value());
6684 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
6685 kDefaultFramerate);
6686
6687 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
6688 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
6689 timestamp_ms += 1000 / kDefaultFramerate;
6690 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6691 WaitForEncodedFrame(timestamp_ms);
6692 }
6693 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
6694 kDefaultFramerate);
6695 // Capture larger frame to trigger a reconfigure.
6696 codec_height_ *= 2;
6697 codec_width_ *= 2;
6698 timestamp_ms += 1000 / kDefaultFramerate;
6699 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6700 WaitForEncodedFrame(timestamp_ms);
6701
6702 EXPECT_EQ(2, sink_.number_of_reconfigurations());
6703 auto current_rate_settings =
6704 fake_encoder_.GetAndResetLastRateControlSettings();
6705 // Ensure we have actually reconfigured twice
6706 // The rate settings should have been set again even though
6707 // they haven't changed.
6708 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006709 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006710
6711 video_stream_encoder_->Stop();
6712}
6713
philipeld9cc8c02019-09-16 14:53:40 +02006714struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02006715 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
6716 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
6717 MOCK_METHOD(void,
6718 RequestEncoderSwitch,
6719 (const webrtc::SdpVideoFormat& format),
6720 (override));
philipeld9cc8c02019-09-16 14:53:40 +02006721};
6722
6723TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
6724 constexpr int kDontCare = 100;
6725
6726 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6727 video_send_config_.encoder_settings.encoder_switch_request_callback =
6728 &switch_callback;
6729 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6730 encoder_config.codec_type = kVideoCodecVP8;
6731 webrtc::test::ScopedFieldTrials field_trial(
6732 "WebRTC-NetworkCondition-EncoderSwitch/"
6733 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6734 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6735
6736 // Reset encoder for new configuration to take effect.
6737 ConfigureEncoder(std::move(encoder_config));
6738
6739 // Send one frame to trigger ReconfigureEncoder.
6740 video_source_.IncomingCapturedFrame(
6741 CreateFrame(kDontCare, kDontCare, kDontCare));
6742
6743 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006744 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6745 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006746 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006747 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006748
Henrik Boström381d1092020-05-12 18:49:07 +02006749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006750 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6751 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6752 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02006753 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006754 /*rtt_ms=*/0,
6755 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006756 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02006757
6758 video_stream_encoder_->Stop();
6759}
6760
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006761TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
6762 constexpr int kDontCare = 100;
6763
6764 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6765 video_send_config_.encoder_settings.encoder_switch_request_callback =
6766 &switch_callback;
6767 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6768 encoder_config.codec_type = kVideoCodecVP8;
6769 webrtc::test::ScopedFieldTrials field_trial(
6770 "WebRTC-NetworkCondition-EncoderSwitch/"
6771 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6772 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6773
6774 // Reset encoder for new configuration to take effect.
6775 ConfigureEncoder(std::move(encoder_config));
6776
6777 // Send one frame to trigger ReconfigureEncoder.
6778 video_source_.IncomingCapturedFrame(
6779 CreateFrame(kDontCare, kDontCare, kDontCare));
6780
6781 using Config = EncoderSwitchRequestCallback::Config;
6782 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
6783 .Times(0);
6784
Henrik Boström381d1092020-05-12 18:49:07 +02006785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006786 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
6787 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
6788 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
6789 /*fraction_lost=*/0,
6790 /*rtt_ms=*/0,
6791 /*cwnd_reduce_ratio=*/0);
6792
6793 video_stream_encoder_->Stop();
6794}
6795
philipeld9cc8c02019-09-16 14:53:40 +02006796TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
6797 constexpr int kSufficientBitrateToNotDrop = 1000;
6798 constexpr int kHighRes = 500;
6799 constexpr int kLowRes = 100;
6800
6801 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6802 video_send_config_.encoder_settings.encoder_switch_request_callback =
6803 &switch_callback;
6804 webrtc::test::ScopedFieldTrials field_trial(
6805 "WebRTC-NetworkCondition-EncoderSwitch/"
6806 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
6807 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6808 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6809 encoder_config.codec_type = kVideoCodecH264;
6810
6811 // Reset encoder for new configuration to take effect.
6812 ConfigureEncoder(std::move(encoder_config));
6813
6814 // The VideoStreamEncoder needs some bitrate before it can start encoding,
6815 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
6816 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02006817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006818 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6819 /*stable_target_bitrate=*/
6820 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6821 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02006822 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006823 /*rtt_ms=*/0,
6824 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02006825
6826 // Send one frame to trigger ReconfigureEncoder.
6827 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
6828 WaitForEncodedFrame(1);
6829
6830 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006831 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6832 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006833 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006834 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006835
6836 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
6837 WaitForEncodedFrame(2);
6838
6839 video_stream_encoder_->Stop();
6840}
6841
philipel9b058032020-02-10 11:30:00 +01006842TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
6843 constexpr int kDontCare = 100;
6844 StrictMock<MockEncoderSelector> encoder_selector;
6845 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6846 &fake_encoder_, &encoder_selector);
6847 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6848
6849 // Reset encoder for new configuration to take effect.
6850 ConfigureEncoder(video_encoder_config_.Copy());
6851
6852 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
6853
6854 video_source_.IncomingCapturedFrame(
6855 CreateFrame(kDontCare, kDontCare, kDontCare));
6856 video_stream_encoder_->Stop();
6857
6858 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
6859 // to it's factory, so in order for the encoder instance in the
6860 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
6861 // reset the |video_stream_encoder_| here.
6862 video_stream_encoder_.reset();
6863}
6864
6865TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
6866 constexpr int kDontCare = 100;
6867
6868 NiceMock<MockEncoderSelector> encoder_selector;
6869 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6870 video_send_config_.encoder_settings.encoder_switch_request_callback =
6871 &switch_callback;
6872 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6873 &fake_encoder_, &encoder_selector);
6874 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6875
6876 // Reset encoder for new configuration to take effect.
6877 ConfigureEncoder(video_encoder_config_.Copy());
6878
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01006879 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01006880 .WillByDefault(Return(SdpVideoFormat("AV1")));
6881 EXPECT_CALL(switch_callback,
6882 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
6883 Field(&SdpVideoFormat::name, "AV1"))));
6884
Henrik Boström381d1092020-05-12 18:49:07 +02006885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006886 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6887 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6888 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01006889 /*fraction_lost=*/0,
6890 /*rtt_ms=*/0,
6891 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006892 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01006893
6894 video_stream_encoder_->Stop();
6895}
6896
6897TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
6898 constexpr int kSufficientBitrateToNotDrop = 1000;
6899 constexpr int kDontCare = 100;
6900
6901 NiceMock<MockVideoEncoder> video_encoder;
6902 NiceMock<MockEncoderSelector> encoder_selector;
6903 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6904 video_send_config_.encoder_settings.encoder_switch_request_callback =
6905 &switch_callback;
6906 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
6907 &video_encoder, &encoder_selector);
6908 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
6909
6910 // Reset encoder for new configuration to take effect.
6911 ConfigureEncoder(video_encoder_config_.Copy());
6912
6913 // The VideoStreamEncoder needs some bitrate before it can start encoding,
6914 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
6915 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02006916 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006917 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6918 /*stable_target_bitrate=*/
6919 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
6920 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01006921 /*fraction_lost=*/0,
6922 /*rtt_ms=*/0,
6923 /*cwnd_reduce_ratio=*/0);
6924
6925 ON_CALL(video_encoder, Encode(_, _))
6926 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
6927 ON_CALL(encoder_selector, OnEncoderBroken())
6928 .WillByDefault(Return(SdpVideoFormat("AV2")));
6929
6930 rtc::Event encode_attempted;
6931 EXPECT_CALL(switch_callback,
6932 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
6933 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
6934 EXPECT_EQ(format.name, "AV2");
6935 encode_attempted.Set();
6936 });
6937
6938 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
6939 encode_attempted.Wait(3000);
6940
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006941 AdvanceTime(TimeDelta::Millis(0));
6942
philipel9b058032020-02-10 11:30:00 +01006943 video_stream_encoder_->Stop();
6944
6945 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
6946 // to it's factory, so in order for the encoder instance in the
6947 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
6948 // reset the |video_stream_encoder_| here.
6949 video_stream_encoder_.reset();
6950}
6951
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006952TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006953 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006954 const int kFrameWidth = 320;
6955 const int kFrameHeight = 180;
6956
6957 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006958 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006959 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006960 /*target_bitrate=*/rate,
6961 /*stable_target_bitrate=*/rate,
6962 /*link_allocation=*/rate,
6963 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006964 /*rtt_ms=*/0,
6965 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006966
6967 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006968 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006969 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6970 frame.set_rotation(kVideoRotation_270);
6971 video_source_.IncomingCapturedFrame(frame);
6972 WaitForEncodedFrame(timestamp_ms);
6973 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6974
6975 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006976 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02006977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006978 /*target_bitrate=*/new_stable_rate,
6979 /*stable_target_bitrate=*/new_stable_rate,
6980 /*link_allocation=*/rate,
6981 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006982 /*rtt_ms=*/0,
6983 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006984 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6985 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
6986 video_stream_encoder_->Stop();
6987}
6988
6989TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006990 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006991 const int kFrameWidth = 320;
6992 const int kFrameHeight = 180;
6993
6994 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006995 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006997 /*target_bitrate=*/rate,
6998 /*stable_target_bitrate=*/rate,
6999 /*link_allocation=*/rate,
7000 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007001 /*rtt_ms=*/0,
7002 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007003
7004 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007005 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007006 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7007 frame.set_rotation(kVideoRotation_270);
7008 video_source_.IncomingCapturedFrame(frame);
7009 WaitForEncodedFrame(timestamp_ms);
7010 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7011
7012 // Set a higher target rate without changing the link_allocation. Should not
7013 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007014 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007015 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007016 /*target_bitrate=*/rate,
7017 /*stable_target_bitrate=*/new_stable_rate,
7018 /*link_allocation=*/rate,
7019 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007020 /*rtt_ms=*/0,
7021 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007022 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7023 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7024 video_stream_encoder_->Stop();
7025}
7026
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007027TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7028 test::ScopedFieldTrials field_trials(
7029 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7030 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7031 const int kFramerateFps = 30;
7032 const int kWidth = 1920;
7033 const int kHeight = 1080;
7034 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7035 // Works on screenshare mode.
7036 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7037 // We rely on the automatic resolution adaptation, but we handle framerate
7038 // adaptation manually by mocking the stats proxy.
7039 video_source_.set_adaptation_enabled(true);
7040
7041 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007043 DataRate::BitsPerSec(kTargetBitrateBps),
7044 DataRate::BitsPerSec(kTargetBitrateBps),
7045 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007046 video_stream_encoder_->SetSource(&video_source_,
7047 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007048 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007049
7050 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7051 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7052
7053 // Pass enough frames with the full update to trigger animation detection.
7054 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007055 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007056 frame.set_ntp_time_ms(timestamp_ms);
7057 frame.set_timestamp_us(timestamp_ms * 1000);
7058 video_source_.IncomingCapturedFrame(frame);
7059 WaitForEncodedFrame(timestamp_ms);
7060 }
7061
7062 // Resolution should be limited.
7063 rtc::VideoSinkWants expected;
7064 expected.max_framerate_fps = kFramerateFps;
7065 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007066 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007067
7068 // Pass one frame with no known update.
7069 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007070 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007071 frame.set_ntp_time_ms(timestamp_ms);
7072 frame.set_timestamp_us(timestamp_ms * 1000);
7073 frame.clear_update_rect();
7074
7075 video_source_.IncomingCapturedFrame(frame);
7076 WaitForEncodedFrame(timestamp_ms);
7077
7078 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007079 EXPECT_THAT(video_source_.sink_wants(),
7080 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007081
7082 video_stream_encoder_->Stop();
7083}
7084
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007085TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7086 const int kWidth = 720; // 540p adapted down.
7087 const int kHeight = 405;
7088 const int kNumFrames = 3;
7089 // Works on screenshare mode.
7090 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7091 /*num_spatial_layers=*/2, /*screenshare=*/true);
7092
7093 video_source_.set_adaptation_enabled(true);
7094
7095 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7096 DataRate::BitsPerSec(kTargetBitrateBps),
7097 DataRate::BitsPerSec(kTargetBitrateBps),
7098 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7099
7100 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7101
7102 // Pass enough frames with the full update to trigger animation detection.
7103 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007104 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007105 frame.set_ntp_time_ms(timestamp_ms);
7106 frame.set_timestamp_us(timestamp_ms * 1000);
7107 video_source_.IncomingCapturedFrame(frame);
7108 WaitForEncodedFrame(timestamp_ms);
7109 }
7110
7111 video_stream_encoder_->Stop();
7112}
7113
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007114TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7115 const float downscale_factors[] = {4.0, 2.0, 1.0};
7116 const int number_layers =
7117 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7118 VideoEncoderConfig config;
7119 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7120 for (int i = 0; i < number_layers; ++i) {
7121 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7122 config.simulcast_layers[i].active = true;
7123 }
7124 config.video_stream_factory =
7125 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7126 "VP8", /*max qp*/ 56, /*screencast*/ false,
7127 /*screenshare enabled*/ false);
7128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7129 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7130 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7131 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7132
7133 // First initialization.
7134 // Encoder should be initialized. Next frame should be key frame.
7135 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7136 sink_.SetNumExpectedLayers(number_layers);
7137 int64_t timestamp_ms = kFrameIntervalMs;
7138 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7139 WaitForEncodedFrame(timestamp_ms);
7140 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7141 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7142 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7143 VideoFrameType::kVideoFrameKey,
7144 VideoFrameType::kVideoFrameKey}));
7145
7146 // Disable top layer.
7147 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7148 config.simulcast_layers[number_layers - 1].active = false;
7149 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7150 sink_.SetNumExpectedLayers(number_layers - 1);
7151 timestamp_ms += kFrameIntervalMs;
7152 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7153 WaitForEncodedFrame(timestamp_ms);
7154 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7155 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7156 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7157 VideoFrameType::kVideoFrameDelta,
7158 VideoFrameType::kVideoFrameDelta}));
7159
7160 // Re-enable top layer.
7161 // Encoder should be re-initialized. Next frame should be key frame.
7162 config.simulcast_layers[number_layers - 1].active = true;
7163 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7164 sink_.SetNumExpectedLayers(number_layers);
7165 timestamp_ms += kFrameIntervalMs;
7166 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7167 WaitForEncodedFrame(timestamp_ms);
7168 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7169 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7170 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7171 VideoFrameType::kVideoFrameKey,
7172 VideoFrameType::kVideoFrameKey}));
7173
7174 // Top layer max rate change.
7175 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7176 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7177 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7178 sink_.SetNumExpectedLayers(number_layers);
7179 timestamp_ms += kFrameIntervalMs;
7180 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7181 WaitForEncodedFrame(timestamp_ms);
7182 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7183 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7184 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7185 VideoFrameType::kVideoFrameDelta,
7186 VideoFrameType::kVideoFrameDelta}));
7187
7188 // Top layer resolution change.
7189 // Encoder should be re-initialized. Next frame should be key frame.
7190 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7191 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7192 sink_.SetNumExpectedLayers(number_layers);
7193 timestamp_ms += kFrameIntervalMs;
7194 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7195 WaitForEncodedFrame(timestamp_ms);
7196 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7197 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7198 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7199 VideoFrameType::kVideoFrameKey,
7200 VideoFrameType::kVideoFrameKey}));
7201 video_stream_encoder_->Stop();
7202}
7203
perkj26091b12016-09-01 01:17:40 -07007204} // namespace webrtc