blob: 7ace082665deec6bf388d325029c64b998e1177f [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010040#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020041#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080043#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020044#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010045#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020046#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020047#include "test/encoder_settings.h"
48#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020049#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010050#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "test/gmock.h"
52#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020053#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020054#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070056
57namespace webrtc {
58
sprang57c2fff2017-01-16 06:24:02 -080059using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020060using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020061using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020062using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020063using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020064using ::testing::Ge;
65using ::testing::Gt;
66using ::testing::Le;
67using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010068using ::testing::Matcher;
69using ::testing::NiceMock;
70using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010071using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020072using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080073
perkj803d97f2016-11-01 11:45:46 -070074namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010076const int kQpLow = 1;
77const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020078const int kMinFramerateFps = 2;
79const int kMinBalancedFramerateFps = 7;
80const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080081const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010082const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020083const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010084const uint32_t kSimulcastTargetBitrateBps = 3150000;
85const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080086const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070087const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020089const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020090const VideoEncoder::ResolutionBitrateLimits
91 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
92const VideoEncoder::ResolutionBitrateLimits
93 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080094
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020095uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
96 0x00, 0x00, 0x03, 0x03, 0xF4,
97 0x05, 0x03, 0xC7, 0xE0, 0x1B,
98 0x41, 0x10, 0x8D, 0x00};
99
perkj803d97f2016-11-01 11:45:46 -0700100class TestBuffer : public webrtc::I420Buffer {
101 public:
102 TestBuffer(rtc::Event* event, int width, int height)
103 : I420Buffer(width, height), event_(event) {}
104
105 private:
106 friend class rtc::RefCountedObject<TestBuffer>;
107 ~TestBuffer() override {
108 if (event_)
109 event_->Set();
110 }
111 rtc::Event* const event_;
112};
113
Noah Richards51db4212019-06-12 06:59:12 -0700114// A fake native buffer that can't be converted to I420.
115class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
116 public:
117 FakeNativeBuffer(rtc::Event* event, int width, int height)
118 : event_(event), width_(width), height_(height) {}
119 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
120 int width() const override { return width_; }
121 int height() const override { return height_; }
122 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
123 return nullptr;
124 }
125
126 private:
127 friend class rtc::RefCountedObject<FakeNativeBuffer>;
128 ~FakeNativeBuffer() override {
129 if (event_)
130 event_->Set();
131 }
132 rtc::Event* const event_;
133 const int width_;
134 const int height_;
135};
136
Evan Shrubsole895556e2020-10-05 09:15:13 +0200137// A fake native buffer that is backed by an NV12 buffer.
138class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
139 public:
140 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
141 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
142
143 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
144 int width() const override { return nv12_buffer_->width(); }
145 int height() const override { return nv12_buffer_->height(); }
146 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
147 return nv12_buffer_->ToI420();
148 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200149 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
150 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
151 if (absl::c_find(types, Type::kNV12) != types.end()) {
152 return nv12_buffer_;
153 }
154 return nullptr;
155 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200156 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
157
158 private:
159 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
160 ~FakeNV12NativeBuffer() override {
161 if (event_)
162 event_->Set();
163 }
164 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
165 rtc::Event* const event_;
166};
167
Niels Möller7dc26b72017-12-06 10:27:48 +0100168class CpuOveruseDetectorProxy : public OveruseFrameDetector {
169 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200170 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
171 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200172 last_target_framerate_fps_(-1),
173 framerate_updated_event_(true /* manual_reset */,
174 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100175 virtual ~CpuOveruseDetectorProxy() {}
176
177 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200178 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100179 last_target_framerate_fps_ = framerate_fps;
180 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200181 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100182 }
183
184 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200185 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100186 return last_target_framerate_fps_;
187 }
188
Niels Möller4db138e2018-04-19 09:04:13 +0200189 CpuOveruseOptions GetOptions() { return options_; }
190
Henrik Boström381d1092020-05-12 18:49:07 +0200191 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
192
Niels Möller7dc26b72017-12-06 10:27:48 +0100193 private:
Markus Handella3765182020-07-08 13:13:32 +0200194 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100195 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200196 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100197};
198
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200199class FakeVideoSourceRestrictionsListener
200 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200201 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200202 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200203 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200204 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200205 RTC_DCHECK(was_restrictions_updated_);
206 }
207
208 rtc::Event* restrictions_updated_event() {
209 return &restrictions_updated_event_;
210 }
211
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200212 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200213 void OnVideoSourceRestrictionsUpdated(
214 VideoSourceRestrictions restrictions,
215 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200216 rtc::scoped_refptr<Resource> reason,
217 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200218 was_restrictions_updated_ = true;
219 restrictions_updated_event_.Set();
220 }
221
222 private:
223 bool was_restrictions_updated_;
224 rtc::Event restrictions_updated_event_;
225};
226
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200227auto WantsFps(Matcher<int> fps_matcher) {
228 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
229 fps_matcher);
230}
231
232auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
233 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
234 AllOf(max_pixel_matcher, Gt(0)));
235}
236
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200237auto ResolutionMax() {
238 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200239 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200240 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
241 Eq(absl::nullopt)));
242}
243
244auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200245 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200246}
247
248auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200249 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200250}
251
252auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200253 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200254}
255
256auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200257 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200258}
259
260auto FpsMaxResolutionMax() {
261 return AllOf(FpsMax(), ResolutionMax());
262}
263
264auto UnlimitedSinkWants() {
265 return AllOf(FpsUnlimited(), ResolutionMax());
266}
267
268auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
269 Matcher<int> fps_range_matcher;
270
271 if (last_frame_pixels <= 320 * 240) {
272 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200273 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200274 fps_range_matcher = AllOf(Ge(10), Le(15));
275 } else if (last_frame_pixels <= 640 * 480) {
276 fps_range_matcher = Ge(15);
277 } else {
278 fps_range_matcher = Eq(kDefaultFramerate);
279 }
280 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
281 fps_range_matcher);
282}
283
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200284auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
285 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
286 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
287}
288
289auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
290 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
291}
292
293auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
294 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
295}
296
297auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
298 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
299 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
300}
301
302auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
303 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
304 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
305}
306
307auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
308 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
309 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
310}
311
312auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
313 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
314 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
315}
316
mflodmancc3d4422017-08-03 08:27:51 -0700317class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700318 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200319 VideoStreamEncoderUnderTest(TimeController* time_controller,
320 TaskQueueFactory* task_queue_factory,
321 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100322 const VideoStreamEncoderSettings& settings,
323 VideoStreamEncoder::BitrateAllocationCallbackType
324 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200325 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100326 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200327 stats_proxy,
328 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200329 std::unique_ptr<OveruseFrameDetector>(
330 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100331 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100332 task_queue_factory,
333 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200334 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200335 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200336 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200337 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200338 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200339 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200340 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200341 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100342 }
perkj803d97f2016-11-01 11:45:46 -0700343
Henrik Boström381d1092020-05-12 18:49:07 +0200344 void SetSourceAndWaitForRestrictionsUpdated(
345 rtc::VideoSourceInterface<VideoFrame>* source,
346 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200347 FakeVideoSourceRestrictionsListener listener;
348 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200349 SetSource(source, degradation_preference);
350 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200351 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200352 }
353
354 void SetSourceAndWaitForFramerateUpdated(
355 rtc::VideoSourceInterface<VideoFrame>* source,
356 const DegradationPreference& degradation_preference) {
357 overuse_detector_proxy_->framerate_updated_event()->Reset();
358 SetSource(source, degradation_preference);
359 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
360 }
361
362 void OnBitrateUpdatedAndWaitForManagedResources(
363 DataRate target_bitrate,
364 DataRate stable_target_bitrate,
365 DataRate link_allocation,
366 uint8_t fraction_lost,
367 int64_t round_trip_time_ms,
368 double cwnd_reduce_ratio) {
369 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
370 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
371 // Bitrate is updated on the encoder queue.
372 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200373 }
374
kthelgason2fc52542017-03-03 00:24:41 -0800375 // This is used as a synchronisation mechanism, to make sure that the
376 // encoder queue is not blocked before we start sending it frames.
377 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100378 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200379 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800380 ASSERT_TRUE(event.Wait(5000));
381 }
382
Henrik Boström91aa7322020-04-28 12:24:33 +0200383 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200384 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200385 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200386 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200387 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200388 event.Set();
389 });
390 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200391 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200392 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200393
Henrik Boström91aa7322020-04-28 12:24:33 +0200394 void TriggerCpuUnderuse() {
395 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200396 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200397 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200398 event.Set();
399 });
400 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200401 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200402 }
kthelgason876222f2016-11-29 01:44:11 -0800403
Henrik Boström91aa7322020-04-28 12:24:33 +0200404 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200405 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200406 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200407 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200408 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200409 event.Set();
410 });
411 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200412 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200413 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200414 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200415 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200416 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200417 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 event.Set();
419 });
420 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200421 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200422 }
423
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200424 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100425 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200426 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
427 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200428 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700429};
430
Noah Richards51db4212019-06-12 06:59:12 -0700431// Simulates simulcast behavior and makes highest stream resolutions divisible
432// by 4.
433class CroppingVideoStreamFactory
434 : public VideoEncoderConfig::VideoStreamFactoryInterface {
435 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200436 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700437
438 private:
439 std::vector<VideoStream> CreateEncoderStreams(
440 int width,
441 int height,
442 const VideoEncoderConfig& encoder_config) override {
443 std::vector<VideoStream> streams = test::CreateVideoStreams(
444 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700445 return streams;
446 }
Noah Richards51db4212019-06-12 06:59:12 -0700447};
448
sprangb1ca0732017-02-01 08:38:12 -0800449class AdaptingFrameForwarder : public test::FrameForwarder {
450 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200451 explicit AdaptingFrameForwarder(TimeController* time_controller)
452 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700453 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800454
455 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200456 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800457 adaptation_enabled_ = enabled;
458 }
459
asaperssonfab67072017-04-04 05:51:49 -0700460 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200461 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800462 return adaptation_enabled_;
463 }
464
Henrik Boström1124ed12021-02-25 10:30:39 +0100465 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
466 // the resolution or frame rate was different than it is currently. If
467 // something else is modified, such as encoder resolutions, but the resolution
468 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700469 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200470 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700471 return last_wants_;
472 }
473
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200474 absl::optional<int> last_sent_width() const { return last_width_; }
475 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800476
sprangb1ca0732017-02-01 08:38:12 -0800477 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200478 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
479 time_controller_->AdvanceTime(TimeDelta::Millis(0));
480
sprangb1ca0732017-02-01 08:38:12 -0800481 int cropped_width = 0;
482 int cropped_height = 0;
483 int out_width = 0;
484 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700485 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200486 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
487 << "w=" << video_frame.width()
488 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700489 if (adapter_.AdaptFrameResolution(
490 video_frame.width(), video_frame.height(),
491 video_frame.timestamp_us() * 1000, &cropped_width,
492 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100493 VideoFrame adapted_frame =
494 VideoFrame::Builder()
495 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
496 nullptr, out_width, out_height))
497 .set_timestamp_rtp(99)
498 .set_timestamp_ms(99)
499 .set_rotation(kVideoRotation_0)
500 .build();
sprangc5d62e22017-04-02 23:53:04 -0700501 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100502 if (video_frame.has_update_rect()) {
503 adapted_frame.set_update_rect(
504 video_frame.update_rect().ScaleWithFrame(
505 video_frame.width(), video_frame.height(), 0, 0,
506 video_frame.width(), video_frame.height(), out_width,
507 out_height));
508 }
sprangc5d62e22017-04-02 23:53:04 -0700509 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800510 last_width_.emplace(adapted_frame.width());
511 last_height_.emplace(adapted_frame.height());
512 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200513 last_width_ = absl::nullopt;
514 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700515 }
sprangb1ca0732017-02-01 08:38:12 -0800516 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200517 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800518 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800519 last_width_.emplace(video_frame.width());
520 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800521 }
522 }
523
524 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
525 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200526 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100527 rtc::VideoSinkWants prev_wants = sink_wants_locked();
528 bool did_adapt =
529 prev_wants.max_pixel_count != wants.max_pixel_count ||
530 prev_wants.target_pixel_count != wants.target_pixel_count ||
531 prev_wants.max_framerate_fps != wants.max_framerate_fps;
532 if (did_adapt) {
533 last_wants_ = prev_wants;
534 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100535 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200536 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800537 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200538
539 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800540 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200541 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
542 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200543 absl::optional<int> last_width_;
544 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800545};
sprangc5d62e22017-04-02 23:53:04 -0700546
Niels Möller213618e2018-07-24 09:29:58 +0200547// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700548class MockableSendStatisticsProxy : public SendStatisticsProxy {
549 public:
550 MockableSendStatisticsProxy(Clock* clock,
551 const VideoSendStream::Config& config,
552 VideoEncoderConfig::ContentType content_type)
553 : SendStatisticsProxy(clock, config, content_type) {}
554
555 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200556 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700557 if (mock_stats_)
558 return *mock_stats_;
559 return SendStatisticsProxy::GetStats();
560 }
561
Niels Möller213618e2018-07-24 09:29:58 +0200562 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200563 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200564 if (mock_stats_)
565 return mock_stats_->input_frame_rate;
566 return SendStatisticsProxy::GetInputFrameRate();
567 }
sprangc5d62e22017-04-02 23:53:04 -0700568 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200569 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700570 mock_stats_.emplace(stats);
571 }
572
573 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200574 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700575 mock_stats_.reset();
576 }
577
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200578 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
579 on_frame_dropped_ = std::move(callback);
580 }
581
sprangc5d62e22017-04-02 23:53:04 -0700582 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200583 void OnFrameDropped(DropReason reason) override {
584 SendStatisticsProxy::OnFrameDropped(reason);
585 if (on_frame_dropped_)
586 on_frame_dropped_(reason);
587 }
588
Markus Handella3765182020-07-08 13:13:32 +0200589 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200590 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200591 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700592};
593
philipel9b058032020-02-10 11:30:00 +0100594class MockEncoderSelector
595 : public VideoEncoderFactory::EncoderSelectorInterface {
596 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200597 MOCK_METHOD(void,
598 OnCurrentEncoder,
599 (const SdpVideoFormat& format),
600 (override));
601 MOCK_METHOD(absl::optional<SdpVideoFormat>,
602 OnAvailableBitrate,
603 (const DataRate& rate),
604 (override));
605 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100606};
607
perkj803d97f2016-11-01 11:45:46 -0700608} // namespace
609
mflodmancc3d4422017-08-03 08:27:51 -0700610class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700611 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200612 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700613
mflodmancc3d4422017-08-03 08:27:51 -0700614 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700615 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700616 codec_width_(320),
617 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200618 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200619 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200620 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700621 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200622 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700623 video_send_config_,
624 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200625 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700626
627 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700628 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700629 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200630 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800631 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200632 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200633 video_send_config_.rtp.payload_name = "FAKE";
634 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700635
Per512ecb32016-09-23 15:52:06 +0200636 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200637 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200638 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
639 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
640 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100641 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700642
Niels Möllerf1338562018-04-26 09:51:47 +0200643 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800644 }
645
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100646 void ConfigureEncoder(
647 VideoEncoderConfig video_encoder_config,
648 VideoStreamEncoder::BitrateAllocationCallbackType
649 allocation_callback_type =
650 VideoStreamEncoder::BitrateAllocationCallbackType::
651 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700652 if (video_stream_encoder_)
653 video_stream_encoder_->Stop();
654 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200655 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100656 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700657 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
658 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700659 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700660 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
661 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200662 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700663 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800664 }
665
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100666 void ResetEncoder(const std::string& payload_name,
667 size_t num_streams,
668 size_t num_temporal_layers,
669 unsigned char num_spatial_layers,
670 bool screenshare,
671 VideoStreamEncoder::BitrateAllocationCallbackType
672 allocation_callback_type =
673 VideoStreamEncoder::BitrateAllocationCallbackType::
674 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200675 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800676
677 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200678 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
679 num_streams, &video_encoder_config);
680 for (auto& layer : video_encoder_config.simulcast_layers) {
681 layer.num_temporal_layers = num_temporal_layers;
682 layer.max_framerate = kDefaultFramerate;
683 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100684 video_encoder_config.max_bitrate_bps =
685 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700686 video_encoder_config.content_type =
687 screenshare ? VideoEncoderConfig::ContentType::kScreen
688 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700689 if (payload_name == "VP9") {
690 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
691 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200692 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700693 video_encoder_config.encoder_specific_settings =
694 new rtc::RefCountedObject<
695 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
696 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100697 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700698 }
699
sprang57c2fff2017-01-16 06:24:02 -0800700 VideoFrame CreateFrame(int64_t ntp_time_ms,
701 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100702 VideoFrame frame =
703 VideoFrame::Builder()
704 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
705 destruction_event, codec_width_, codec_height_))
706 .set_timestamp_rtp(99)
707 .set_timestamp_ms(99)
708 .set_rotation(kVideoRotation_0)
709 .build();
sprang57c2fff2017-01-16 06:24:02 -0800710 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700711 return frame;
712 }
713
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100714 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
715 rtc::Event* destruction_event,
716 int offset_x) const {
717 VideoFrame frame =
718 VideoFrame::Builder()
719 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
720 destruction_event, codec_width_, codec_height_))
721 .set_timestamp_rtp(99)
722 .set_timestamp_ms(99)
723 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100724 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100725 .build();
726 frame.set_ntp_time_ms(ntp_time_ms);
727 return frame;
728 }
729
sprang57c2fff2017-01-16 06:24:02 -0800730 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100731 VideoFrame frame =
732 VideoFrame::Builder()
733 .set_video_frame_buffer(
734 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
735 .set_timestamp_rtp(99)
736 .set_timestamp_ms(99)
737 .set_rotation(kVideoRotation_0)
738 .build();
sprang57c2fff2017-01-16 06:24:02 -0800739 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700740 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700741 return frame;
742 }
743
Evan Shrubsole895556e2020-10-05 09:15:13 +0200744 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
745 VideoFrame frame =
746 VideoFrame::Builder()
747 .set_video_frame_buffer(NV12Buffer::Create(width, height))
748 .set_timestamp_rtp(99)
749 .set_timestamp_ms(99)
750 .set_rotation(kVideoRotation_0)
751 .build();
752 frame.set_ntp_time_ms(ntp_time_ms);
753 frame.set_timestamp_us(ntp_time_ms * 1000);
754 return frame;
755 }
756
Noah Richards51db4212019-06-12 06:59:12 -0700757 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
758 rtc::Event* destruction_event,
759 int width,
760 int height) const {
761 VideoFrame frame =
762 VideoFrame::Builder()
763 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
764 destruction_event, width, height))
765 .set_timestamp_rtp(99)
766 .set_timestamp_ms(99)
767 .set_rotation(kVideoRotation_0)
768 .build();
769 frame.set_ntp_time_ms(ntp_time_ms);
770 return frame;
771 }
772
Evan Shrubsole895556e2020-10-05 09:15:13 +0200773 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
774 rtc::Event* destruction_event,
775 int width,
776 int height) const {
777 VideoFrame frame = VideoFrame::Builder()
778 .set_video_frame_buffer(
779 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
780 destruction_event, width, height))
781 .set_timestamp_rtp(99)
782 .set_timestamp_ms(99)
783 .set_rotation(kVideoRotation_0)
784 .build();
785 frame.set_ntp_time_ms(ntp_time_ms);
786 return frame;
787 }
788
Noah Richards51db4212019-06-12 06:59:12 -0700789 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
790 rtc::Event* destruction_event) const {
791 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
792 codec_height_);
793 }
794
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100795 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100797 DataRate::BitsPerSec(kTargetBitrateBps),
798 DataRate::BitsPerSec(kTargetBitrateBps),
799 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100800
801 video_source_.IncomingCapturedFrame(
802 CreateFrame(1, codec_width_, codec_height_));
803 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200804 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100805 }
806
sprang4847ae62017-06-27 07:06:52 -0700807 void WaitForEncodedFrame(int64_t expected_ntp_time) {
808 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200809 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700810 }
811
812 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
813 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200814 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700815 return ok;
816 }
817
818 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
819 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200820 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700821 }
822
823 void ExpectDroppedFrame() {
824 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200825 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700826 }
827
828 bool WaitForFrame(int64_t timeout_ms) {
829 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200830 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700831 return ok;
832 }
833
perkj26091b12016-09-01 01:17:40 -0700834 class TestEncoder : public test::FakeEncoder {
835 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200836 explicit TestEncoder(TimeController* time_controller)
837 : FakeEncoder(time_controller->GetClock()),
838 time_controller_(time_controller) {
839 RTC_DCHECK(time_controller_);
840 }
perkj26091b12016-09-01 01:17:40 -0700841
asaperssonfab67072017-04-04 05:51:49 -0700842 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200843 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700844 return config_;
845 }
846
847 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200848 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700849 block_next_encode_ = true;
850 }
851
Erik Språngaed30702018-11-05 12:57:17 +0100852 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200853 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200854 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100855 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100856 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100857 info.scaling_settings = VideoEncoder::ScalingSettings(
858 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100859 }
860 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100861 for (int i = 0; i < kMaxSpatialLayers; ++i) {
862 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100863 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100864 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100865 for (int tid = 0; tid < num_layers; ++tid)
866 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100867 }
868 }
Erik Språngaed30702018-11-05 12:57:17 +0100869 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200870
871 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100872 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200873 info.apply_alignment_to_all_simulcast_layers =
874 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200875 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100876 return info;
kthelgason876222f2016-11-29 01:44:11 -0800877 }
878
Erik Språngb7cb7b52019-02-26 15:52:33 +0100879 int32_t RegisterEncodeCompleteCallback(
880 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200881 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100882 encoded_image_callback_ = callback;
883 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
884 }
885
perkjfa10b552016-10-02 23:45:26 -0700886 void ContinueEncode() { continue_encode_event_.Set(); }
887
888 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
889 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200890 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700891 EXPECT_EQ(timestamp_, timestamp);
892 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
893 }
894
kthelgason2fc52542017-03-03 00:24:41 -0800895 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200896 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800897 quality_scaling_ = b;
898 }
kthelgasonad9010c2017-02-14 00:46:51 -0800899
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100900 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200901 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100902 requested_resolution_alignment_ = requested_resolution_alignment;
903 }
904
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200905 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
906 MutexLock lock(&local_mutex_);
907 apply_alignment_to_all_simulcast_layers_ = b;
908 }
909
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100910 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200911 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100912 is_hardware_accelerated_ = is_hardware_accelerated;
913 }
914
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100915 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
916 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200917 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100918 temporal_layers_supported_[spatial_idx] = supported;
919 }
920
Sergey Silkin6456e352019-07-08 17:56:40 +0200921 void SetResolutionBitrateLimits(
922 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200923 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200924 resolution_bitrate_limits_ = thresholds;
925 }
926
sprangfe627f32017-03-29 08:24:59 -0700927 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200928 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700929 force_init_encode_failed_ = force_failure;
930 }
931
Niels Möller6bb5ab92019-01-11 11:11:10 +0100932 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200933 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100934 rate_factor_ = rate_factor;
935 }
936
Erik Språngd7329ca2019-02-21 21:19:53 +0100937 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200938 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100939 return last_framerate_;
940 }
941
Erik Språngd7329ca2019-02-21 21:19:53 +0100942 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200943 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100944 return last_update_rect_;
945 }
946
Niels Möller87e2d782019-03-07 10:18:23 +0100947 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200948 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100949 return last_frame_types_;
950 }
951
952 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100953 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100954 keyframe ? VideoFrameType::kVideoFrameKey
955 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100956 {
Markus Handella3765182020-07-08 13:13:32 +0200957 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100958 last_frame_types_ = frame_type;
959 }
Niels Möllerb859b322019-03-07 12:40:01 +0100960 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100961 }
962
Sergey Silkin727d2af2021-03-11 17:05:44 +0000963 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200964 MutexLock lock(&local_mutex_);
Sergey Silkin727d2af2021-03-11 17:05:44 +0000965 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100966 }
967
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200968 void SetEncodedImageData(
969 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200970 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200971 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200972 }
973
Erik Språngd7329ca2019-02-21 21:19:53 +0100974 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200975 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100976 expect_null_frame_ = true;
977 }
978
Erik Språng5056af02019-09-02 15:53:11 +0200979 absl::optional<VideoEncoder::RateControlParameters>
980 GetAndResetLastRateControlSettings() {
981 auto settings = last_rate_control_settings_;
982 last_rate_control_settings_.reset();
983 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100984 }
985
Evan Shrubsole895556e2020-10-05 09:15:13 +0200986 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
987 MutexLock lock(&local_mutex_);
988 return last_input_pixel_format_;
989 }
990
Sergey Silkin5ee69672019-07-02 14:18:34 +0200991 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200992 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200993 return num_encoder_initializations_;
994 }
995
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200996 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200997 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200998 return num_set_rates_;
999 }
1000
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001001 VideoCodec video_codec() const {
1002 MutexLock lock(&local_mutex_);
1003 return video_codec_;
1004 }
1005
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001006 void SetPreferredPixelFormats(
1007 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1008 pixel_formats) {
1009 MutexLock lock(&local_mutex_);
1010 preferred_pixel_formats_ = std::move(pixel_formats);
1011 }
1012
perkjfa10b552016-10-02 23:45:26 -07001013 private:
perkj26091b12016-09-01 01:17:40 -07001014 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001015 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001016 bool block_encode;
1017 {
Markus Handella3765182020-07-08 13:13:32 +02001018 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001019 if (expect_null_frame_) {
1020 EXPECT_EQ(input_image.timestamp(), 0u);
1021 EXPECT_EQ(input_image.width(), 1);
1022 last_frame_types_ = *frame_types;
1023 expect_null_frame_ = false;
1024 } else {
1025 EXPECT_GT(input_image.timestamp(), timestamp_);
1026 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1027 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1028 }
perkj26091b12016-09-01 01:17:40 -07001029
1030 timestamp_ = input_image.timestamp();
1031 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001032 last_input_width_ = input_image.width();
1033 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001034 block_encode = block_next_encode_;
1035 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001036 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001037 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001038 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001039 }
Niels Möllerb859b322019-03-07 12:40:01 +01001040 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001041 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001042 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001043
perkj26091b12016-09-01 01:17:40 -07001044 return result;
1045 }
1046
Niels Möller08ae7ce2020-09-23 15:58:12 +02001047 CodecSpecificInfo EncodeHook(
1048 EncodedImage& encoded_image,
1049 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001050 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001051 {
1052 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001053 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001054 }
1055 MutexLock lock(&local_mutex_);
1056 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001057 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001058 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001059 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001060 }
1061
sprangfe627f32017-03-29 08:24:59 -07001062 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001063 const Settings& settings) override {
1064 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001065
Markus Handella3765182020-07-08 13:13:32 +02001066 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001067 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001068
1069 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001070 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001071
Erik Språng82fad3d2018-03-21 09:57:23 +01001072 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001073 // Simulate setting up temporal layers, in order to validate the life
1074 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001075 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001076 frame_buffer_controller_ =
1077 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001078 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001079 if (force_init_encode_failed_) {
1080 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001081 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001082 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001083
Erik Språngb7cb7b52019-02-26 15:52:33 +01001084 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001085 return res;
1086 }
1087
Erik Språngb7cb7b52019-02-26 15:52:33 +01001088 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001089 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001090 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1091 initialized_ = EncoderState::kUninitialized;
1092 return FakeEncoder::Release();
1093 }
1094
Erik Språng16cb8f52019-04-12 13:59:09 +02001095 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001096 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001097 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001098 VideoBitrateAllocation adjusted_rate_allocation;
1099 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1100 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001101 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001102 adjusted_rate_allocation.SetBitrate(
1103 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001104 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001105 rate_factor_));
1106 }
1107 }
1108 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001109 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001110 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001111 RateControlParameters adjusted_paramters = parameters;
1112 adjusted_paramters.bitrate = adjusted_rate_allocation;
1113 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001114 }
1115
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001116 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001117 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001118 enum class EncoderState {
1119 kUninitialized,
1120 kInitializationFailed,
1121 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001122 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1123 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001124 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001125 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1126 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1127 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1128 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1129 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1130 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001131 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1132 false;
Markus Handella3765182020-07-08 13:13:32 +02001133 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001134 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1135 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001136 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001137 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001138 absl::optional<bool>
1139 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001140 local_mutex_);
1141 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1142 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1143 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001144 absl::optional<VideoEncoder::RateControlParameters>
1145 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001146 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1147 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001148 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001149 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001150 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1151 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001152 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001153 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001154 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001155 RTC_GUARDED_BY(local_mutex_);
1156 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001157 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001158 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1159 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001160 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1161 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001162 };
1163
mflodmancc3d4422017-08-03 08:27:51 -07001164 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001165 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001166 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1167 : time_controller_(time_controller), test_encoder_(test_encoder) {
1168 RTC_DCHECK(time_controller_);
1169 }
perkj26091b12016-09-01 01:17:40 -07001170
perkj26091b12016-09-01 01:17:40 -07001171 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001172 EXPECT_TRUE(
1173 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1174 }
1175
1176 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1177 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001178 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001179 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001180 return false;
perkj26091b12016-09-01 01:17:40 -07001181 {
Markus Handella3765182020-07-08 13:13:32 +02001182 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001183 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001184 }
1185 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001186 return true;
perkj26091b12016-09-01 01:17:40 -07001187 }
1188
sprangb1ca0732017-02-01 08:38:12 -08001189 void WaitForEncodedFrame(uint32_t expected_width,
1190 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001191 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001192 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001193 }
1194
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001195 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001196 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001197 uint32_t width = 0;
1198 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001199 {
Markus Handella3765182020-07-08 13:13:32 +02001200 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001201 width = last_width_;
1202 height = last_height_;
1203 }
1204 EXPECT_EQ(expected_height, height);
1205 EXPECT_EQ(expected_width, width);
1206 }
1207
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001208 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1209 VideoRotation rotation;
1210 {
Markus Handella3765182020-07-08 13:13:32 +02001211 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001212 rotation = last_rotation_;
1213 }
1214 EXPECT_EQ(expected_rotation, rotation);
1215 }
1216
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001217 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001218
sprangc5d62e22017-04-02 23:53:04 -07001219 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001220 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1221 bool ret = encoded_frame_event_.Wait(timeout_ms);
1222 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1223 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001224 }
1225
perkj26091b12016-09-01 01:17:40 -07001226 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001227 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001228 expect_frames_ = false;
1229 }
1230
asaperssonfab67072017-04-04 05:51:49 -07001231 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001232 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001233 return number_of_reconfigurations_;
1234 }
1235
asaperssonfab67072017-04-04 05:51:49 -07001236 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001237 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001238 return min_transmit_bitrate_bps_;
1239 }
1240
Erik Språngd7329ca2019-02-21 21:19:53 +01001241 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001242 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001243 num_expected_layers_ = num_layers;
1244 }
1245
Erik Språngb7cb7b52019-02-26 15:52:33 +01001246 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001247 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001248 return last_capture_time_ms_;
1249 }
1250
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001251 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001252 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001253 return std::move(last_encoded_image_data_);
1254 }
1255
Per Kjellanderdcef6412020-10-07 15:09:05 +02001256 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1257 MutexLock lock(&mutex_);
1258 return last_bitrate_allocation_;
1259 }
1260
1261 int number_of_bitrate_allocations() const {
1262 MutexLock lock(&mutex_);
1263 return number_of_bitrate_allocations_;
1264 }
1265
Per Kjellandera9434842020-10-15 17:53:22 +02001266 VideoLayersAllocation GetLastVideoLayersAllocation() {
1267 MutexLock lock(&mutex_);
1268 return last_layers_allocation_;
1269 }
1270
1271 int number_of_layers_allocations() const {
1272 MutexLock lock(&mutex_);
1273 return number_of_layers_allocations_;
1274 }
1275
perkj26091b12016-09-01 01:17:40 -07001276 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001277 Result OnEncodedImage(
1278 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001279 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001280 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001281 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001282 last_encoded_image_data_ = std::vector<uint8_t>(
1283 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001284 uint32_t timestamp = encoded_image.Timestamp();
1285 if (last_timestamp_ != timestamp) {
1286 num_received_layers_ = 1;
1287 } else {
1288 ++num_received_layers_;
1289 }
1290 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001291 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001292 last_width_ = encoded_image._encodedWidth;
1293 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001294 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001295 if (num_received_layers_ == num_expected_layers_) {
1296 encoded_frame_event_.Set();
1297 }
sprangb1ca0732017-02-01 08:38:12 -08001298 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001299 }
1300
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001301 void OnEncoderConfigurationChanged(
1302 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001303 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001304 VideoEncoderConfig::ContentType content_type,
1305 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001306 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001307 ++number_of_reconfigurations_;
1308 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1309 }
1310
Per Kjellanderdcef6412020-10-07 15:09:05 +02001311 void OnBitrateAllocationUpdated(
1312 const VideoBitrateAllocation& allocation) override {
1313 MutexLock lock(&mutex_);
1314 ++number_of_bitrate_allocations_;
1315 last_bitrate_allocation_ = allocation;
1316 }
1317
Per Kjellandera9434842020-10-15 17:53:22 +02001318 void OnVideoLayersAllocationUpdated(
1319 VideoLayersAllocation allocation) override {
1320 MutexLock lock(&mutex_);
1321 ++number_of_layers_allocations_;
1322 last_layers_allocation_ = allocation;
1323 rtc::StringBuilder log;
1324 for (const auto& layer : allocation.active_spatial_layers) {
1325 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1326 << "[";
1327 for (const auto target_bitrate :
1328 layer.target_bitrate_per_temporal_layer) {
1329 log << target_bitrate.kbps() << ",";
1330 }
1331 log << "]";
1332 }
1333 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1334 }
1335
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001336 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001337 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001338 TestEncoder* test_encoder_;
1339 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001340 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001341 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001342 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001343 uint32_t last_height_ = 0;
1344 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001345 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001346 size_t num_expected_layers_ = 1;
1347 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001348 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001349 int number_of_reconfigurations_ = 0;
1350 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001351 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1352 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001353 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1354 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001355 };
1356
Sergey Silkin5ee69672019-07-02 14:18:34 +02001357 class VideoBitrateAllocatorProxyFactory
1358 : public VideoBitrateAllocatorFactory {
1359 public:
1360 VideoBitrateAllocatorProxyFactory()
1361 : bitrate_allocator_factory_(
1362 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1363
1364 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1365 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001366 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001367 codec_config_ = codec;
1368 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1369 }
1370
1371 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001372 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001373 return codec_config_;
1374 }
1375
1376 private:
1377 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1378
Markus Handella3765182020-07-08 13:13:32 +02001379 mutable Mutex mutex_;
1380 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001381 };
1382
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001383 Clock* clock() { return time_controller_.GetClock(); }
1384 void AdvanceTime(TimeDelta duration) {
1385 time_controller_.AdvanceTime(duration);
1386 }
1387
1388 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1389
1390 protected:
1391 virtual TaskQueueFactory* GetTaskQueueFactory() {
1392 return time_controller_.GetTaskQueueFactory();
1393 }
1394
1395 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001396 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001397 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001398 int codec_width_;
1399 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001400 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001401 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001402 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001403 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001404 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001405 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001406 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001407 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001408};
1409
mflodmancc3d4422017-08-03 08:27:51 -07001410TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001411 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001412 DataRate::BitsPerSec(kTargetBitrateBps),
1413 DataRate::BitsPerSec(kTargetBitrateBps),
1414 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001415 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001416 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001417 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001418 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001419 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001420}
1421
mflodmancc3d4422017-08-03 08:27:51 -07001422TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001423 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001424 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001425 // The encoder will cache up to one frame for a short duration. Adding two
1426 // frames means that the first frame will be dropped and the second frame will
1427 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001428 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001429 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001430 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001431 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001432
Henrik Boström381d1092020-05-12 18:49:07 +02001433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001434 DataRate::BitsPerSec(kTargetBitrateBps),
1435 DataRate::BitsPerSec(kTargetBitrateBps),
1436 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001437
Sebastian Janssona3177052018-04-10 13:05:49 +02001438 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001439 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001440 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1441
1442 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001443 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001444}
1445
mflodmancc3d4422017-08-03 08:27:51 -07001446TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001448 DataRate::BitsPerSec(kTargetBitrateBps),
1449 DataRate::BitsPerSec(kTargetBitrateBps),
1450 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001451 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001452 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001453
Henrik Boström381d1092020-05-12 18:49:07 +02001454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1455 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1456 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001457 // The encoder will cache up to one frame for a short duration. Adding two
1458 // frames means that the first frame will be dropped and the second frame will
1459 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001460 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001461 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001462
Henrik Boström381d1092020-05-12 18:49:07 +02001463 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001464 DataRate::BitsPerSec(kTargetBitrateBps),
1465 DataRate::BitsPerSec(kTargetBitrateBps),
1466 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001467 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001468 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1469 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001470 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001471}
1472
mflodmancc3d4422017-08-03 08:27:51 -07001473TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001475 DataRate::BitsPerSec(kTargetBitrateBps),
1476 DataRate::BitsPerSec(kTargetBitrateBps),
1477 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001478 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001479 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001480
1481 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001482 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001483
perkja49cbd32016-09-16 07:53:41 -07001484 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001485 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001486 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001487}
1488
mflodmancc3d4422017-08-03 08:27:51 -07001489TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001490 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001491 DataRate::BitsPerSec(kTargetBitrateBps),
1492 DataRate::BitsPerSec(kTargetBitrateBps),
1493 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001494
perkja49cbd32016-09-16 07:53:41 -07001495 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001496 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001497
mflodmancc3d4422017-08-03 08:27:51 -07001498 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001499 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001500 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001501 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1502 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001503}
1504
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001505class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1506 public:
1507 VideoStreamEncoderBlockedTest() {}
1508
1509 TaskQueueFactory* GetTaskQueueFactory() override {
1510 return task_queue_factory_.get();
1511 }
1512
1513 private:
1514 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1515 CreateDefaultTaskQueueFactory();
1516};
1517
1518TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001519 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001520 DataRate::BitsPerSec(kTargetBitrateBps),
1521 DataRate::BitsPerSec(kTargetBitrateBps),
1522 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001523
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001524 int dropped_count = 0;
1525 stats_proxy_->SetDroppedFrameCallback(
1526 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1527 ++dropped_count;
1528 });
1529
perkj26091b12016-09-01 01:17:40 -07001530 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001531 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001532 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001533 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1534 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001535 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1536 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001537 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001538 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001539
mflodmancc3d4422017-08-03 08:27:51 -07001540 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001541
1542 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001543}
1544
Noah Richards51db4212019-06-12 06:59:12 -07001545TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001547 DataRate::BitsPerSec(kTargetBitrateBps),
1548 DataRate::BitsPerSec(kTargetBitrateBps),
1549 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001550
1551 rtc::Event frame_destroyed_event;
1552 video_source_.IncomingCapturedFrame(
1553 CreateFakeNativeFrame(1, &frame_destroyed_event));
1554 ExpectDroppedFrame();
1555 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1556 video_stream_encoder_->Stop();
1557}
1558
1559TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1560 // Use the cropping factory.
1561 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001562 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001563 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1564 kMaxPayloadLength);
1565 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1566
1567 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001569 DataRate::BitsPerSec(kTargetBitrateBps),
1570 DataRate::BitsPerSec(kTargetBitrateBps),
1571 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001572 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1573 WaitForEncodedFrame(1);
1574 // The encoder will have been configured once.
1575 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1576 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1577 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1578
1579 // Now send in a fake frame that needs to be cropped as the width/height
1580 // aren't divisible by 4 (see CreateEncoderStreams above).
1581 rtc::Event frame_destroyed_event;
1582 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1583 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1584 ExpectDroppedFrame();
1585 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1586 video_stream_encoder_->Stop();
1587}
1588
Evan Shrubsole895556e2020-10-05 09:15:13 +02001589TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1591 DataRate::BitsPerSec(kTargetBitrateBps),
1592 DataRate::BitsPerSec(kTargetBitrateBps),
1593 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1594
1595 video_source_.IncomingCapturedFrame(
1596 CreateNV12Frame(1, codec_width_, codec_height_));
1597 WaitForEncodedFrame(1);
1598 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1599 fake_encoder_.GetLastInputPixelFormat());
1600 video_stream_encoder_->Stop();
1601}
1602
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001603TEST_F(VideoStreamEncoderTest,
1604 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1606 DataRate::BitsPerSec(kTargetBitrateBps),
1607 DataRate::BitsPerSec(kTargetBitrateBps),
1608 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1609
1610 fake_encoder_.SetPreferredPixelFormats({});
1611
1612 rtc::Event frame_destroyed_event;
1613 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1614 1, &frame_destroyed_event, codec_width_, codec_height_));
1615 WaitForEncodedFrame(1);
1616 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1617 fake_encoder_.GetLastInputPixelFormat());
1618 video_stream_encoder_->Stop();
1619}
1620
1621TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1622 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1623 DataRate::BitsPerSec(kTargetBitrateBps),
1624 DataRate::BitsPerSec(kTargetBitrateBps),
1625 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1626
1627 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1628
1629 rtc::Event frame_destroyed_event;
1630 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1631 1, &frame_destroyed_event, codec_width_, codec_height_));
1632 WaitForEncodedFrame(1);
1633 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1634 fake_encoder_.GetLastInputPixelFormat());
1635 video_stream_encoder_->Stop();
1636}
1637
1638TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1640 DataRate::BitsPerSec(kTargetBitrateBps),
1641 DataRate::BitsPerSec(kTargetBitrateBps),
1642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1643
1644 // Fake NV12 native frame does not allow mapping to I444.
1645 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1646
1647 rtc::Event frame_destroyed_event;
1648 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1649 1, &frame_destroyed_event, codec_width_, codec_height_));
1650 WaitForEncodedFrame(1);
1651 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1652 fake_encoder_.GetLastInputPixelFormat());
1653 video_stream_encoder_->Stop();
1654}
1655
Evan Shrubsole895556e2020-10-05 09:15:13 +02001656TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1658 DataRate::BitsPerSec(kTargetBitrateBps),
1659 DataRate::BitsPerSec(kTargetBitrateBps),
1660 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1661
1662 rtc::Event frame_destroyed_event;
1663 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1664 1, &frame_destroyed_event, codec_width_, codec_height_));
1665 WaitForEncodedFrame(1);
1666 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1667 fake_encoder_.GetLastInputPixelFormat());
1668 video_stream_encoder_->Stop();
1669}
1670
Ying Wang9b881ab2020-02-07 14:29:32 +01001671TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001673 DataRate::BitsPerSec(kTargetBitrateBps),
1674 DataRate::BitsPerSec(kTargetBitrateBps),
1675 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001676 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1677 WaitForEncodedFrame(1);
1678
Henrik Boström381d1092020-05-12 18:49:07 +02001679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001680 DataRate::BitsPerSec(kTargetBitrateBps),
1681 DataRate::BitsPerSec(kTargetBitrateBps),
1682 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001683 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1684 // frames. Adding two frames means that the first frame will be dropped and
1685 // the second frame will be sent to the encoder.
1686 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1687 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1688 WaitForEncodedFrame(3);
1689 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1690 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1691 WaitForEncodedFrame(5);
1692 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1693 video_stream_encoder_->Stop();
1694}
1695
mflodmancc3d4422017-08-03 08:27:51 -07001696TEST_F(VideoStreamEncoderTest,
1697 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001699 DataRate::BitsPerSec(kTargetBitrateBps),
1700 DataRate::BitsPerSec(kTargetBitrateBps),
1701 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001702 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001703
1704 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001705 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001706 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001707 // The encoder will have been configured once when the first frame is
1708 // received.
1709 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001710
1711 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001712 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001713 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001714 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001715 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001716
1717 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001718 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001719 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001720 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001721 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001722
mflodmancc3d4422017-08-03 08:27:51 -07001723 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001724}
1725
mflodmancc3d4422017-08-03 08:27:51 -07001726TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001728 DataRate::BitsPerSec(kTargetBitrateBps),
1729 DataRate::BitsPerSec(kTargetBitrateBps),
1730 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001731
1732 // Capture a frame and wait for it to synchronize with the encoder thread.
1733 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001734 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001735 // The encoder will have been configured once.
1736 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001737 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1738 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1739
1740 codec_width_ *= 2;
1741 codec_height_ *= 2;
1742 // Capture a frame with a higher resolution and wait for it to synchronize
1743 // with the encoder thread.
1744 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001745 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001746 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1747 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001748 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001749
mflodmancc3d4422017-08-03 08:27:51 -07001750 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001751}
1752
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001753TEST_F(VideoStreamEncoderTest,
1754 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001756 DataRate::BitsPerSec(kTargetBitrateBps),
1757 DataRate::BitsPerSec(kTargetBitrateBps),
1758 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001759
1760 // Capture a frame and wait for it to synchronize with the encoder thread.
1761 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1762 WaitForEncodedFrame(1);
1763
1764 VideoEncoderConfig video_encoder_config;
1765 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1766 // Changing the max payload data length recreates encoder.
1767 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1768 kMaxPayloadLength / 2);
1769
1770 // Capture a frame and wait for it to synchronize with the encoder thread.
1771 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1772 WaitForEncodedFrame(2);
1773 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1774
1775 video_stream_encoder_->Stop();
1776}
1777
Sergey Silkin5ee69672019-07-02 14:18:34 +02001778TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001780 DataRate::BitsPerSec(kTargetBitrateBps),
1781 DataRate::BitsPerSec(kTargetBitrateBps),
1782 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001783
1784 VideoEncoderConfig video_encoder_config;
1785 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1786 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1787 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1788 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1789 kMaxPayloadLength);
1790
1791 // Capture a frame and wait for it to synchronize with the encoder thread.
1792 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1793 WaitForEncodedFrame(1);
1794 // The encoder will have been configured once when the first frame is
1795 // received.
1796 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1797 EXPECT_EQ(kTargetBitrateBps,
1798 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1799 EXPECT_EQ(kStartBitrateBps,
1800 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1801
Sergey Silkin6456e352019-07-08 17:56:40 +02001802 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1803 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001804 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1805 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1806 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1807 kMaxPayloadLength);
1808
1809 // Capture a frame and wait for it to synchronize with the encoder thread.
1810 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1811 WaitForEncodedFrame(2);
1812 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1813 // Bitrate limits have changed - rate allocator should be reconfigured,
1814 // encoder should not be reconfigured.
1815 EXPECT_EQ(kTargetBitrateBps * 2,
1816 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1817 EXPECT_EQ(kStartBitrateBps * 2,
1818 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1819 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1820
1821 video_stream_encoder_->Stop();
1822}
1823
Sergey Silkin6456e352019-07-08 17:56:40 +02001824TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001825 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001827 DataRate::BitsPerSec(kTargetBitrateBps),
1828 DataRate::BitsPerSec(kTargetBitrateBps),
1829 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001830
Sergey Silkincd02eba2020-01-20 14:48:40 +01001831 const uint32_t kMinEncBitrateKbps = 100;
1832 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001833 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001834 /*frame_size_pixels=*/codec_width_ * codec_height_,
1835 /*min_start_bitrate_bps=*/0,
1836 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1837 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001838 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1839
Sergey Silkincd02eba2020-01-20 14:48:40 +01001840 VideoEncoderConfig video_encoder_config;
1841 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1842 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1843 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1844 (kMinEncBitrateKbps + 1) * 1000;
1845 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1846 kMaxPayloadLength);
1847
1848 // When both encoder and app provide bitrate limits, the intersection of
1849 // provided sets should be used.
1850 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1851 WaitForEncodedFrame(1);
1852 EXPECT_EQ(kMaxEncBitrateKbps,
1853 bitrate_allocator_factory_.codec_config().maxBitrate);
1854 EXPECT_EQ(kMinEncBitrateKbps + 1,
1855 bitrate_allocator_factory_.codec_config().minBitrate);
1856
1857 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1858 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1859 (kMinEncBitrateKbps - 1) * 1000;
1860 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1861 kMaxPayloadLength);
1862 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001863 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001864 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001865 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001866 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001867 bitrate_allocator_factory_.codec_config().minBitrate);
1868
Sergey Silkincd02eba2020-01-20 14:48:40 +01001869 video_stream_encoder_->Stop();
1870}
1871
1872TEST_F(VideoStreamEncoderTest,
1873 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001874 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001875 DataRate::BitsPerSec(kTargetBitrateBps),
1876 DataRate::BitsPerSec(kTargetBitrateBps),
1877 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001878
1879 const uint32_t kMinAppBitrateKbps = 100;
1880 const uint32_t kMaxAppBitrateKbps = 200;
1881 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1882 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1883 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1884 /*frame_size_pixels=*/codec_width_ * codec_height_,
1885 /*min_start_bitrate_bps=*/0,
1886 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1887 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1888 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1889
1890 VideoEncoderConfig video_encoder_config;
1891 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1892 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1893 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1894 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001895 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1896 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001897
Sergey Silkincd02eba2020-01-20 14:48:40 +01001898 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1899 WaitForEncodedFrame(1);
1900 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001901 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001902 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001903 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001904
1905 video_stream_encoder_->Stop();
1906}
1907
1908TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001909 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001911 DataRate::BitsPerSec(kTargetBitrateBps),
1912 DataRate::BitsPerSec(kTargetBitrateBps),
1913 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001914
1915 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001916 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001917 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001918 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001919 fake_encoder_.SetResolutionBitrateLimits(
1920 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1921
1922 VideoEncoderConfig video_encoder_config;
1923 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1924 video_encoder_config.max_bitrate_bps = 0;
1925 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1926 kMaxPayloadLength);
1927
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001928 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001929 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1930 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001931 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1932 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001933 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1934 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1935
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001936 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001937 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1938 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001939 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1940 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001941 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1942 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1943
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001944 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001945 // encoder for 360p should be used.
1946 video_source_.IncomingCapturedFrame(
1947 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1948 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1950 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001951 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1952 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1953
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001954 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001955 // ignored.
1956 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1957 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001958 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1959 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001960 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1961 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1963 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001964 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1965 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1966
1967 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1968 // for 270p should be used.
1969 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1970 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001971 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1972 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001973 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1974 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1975
1976 video_stream_encoder_->Stop();
1977}
1978
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001979TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001981 DataRate::BitsPerSec(kTargetBitrateBps),
1982 DataRate::BitsPerSec(kTargetBitrateBps),
1983 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001984
1985 VideoEncoderConfig video_encoder_config;
1986 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1987 video_encoder_config.max_bitrate_bps = 0;
1988 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1989 kMaxPayloadLength);
1990
1991 // Encode 720p frame to get the default encoder target bitrate.
1992 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1993 WaitForEncodedFrame(1);
1994 const uint32_t kDefaultTargetBitrateFor720pKbps =
1995 bitrate_allocator_factory_.codec_config()
1996 .simulcastStream[0]
1997 .targetBitrate;
1998
1999 // Set the max recommended encoder bitrate to something lower than the default
2000 // target bitrate.
2001 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2002 1280 * 720, 10 * 1000, 10 * 1000,
2003 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2004 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2005
2006 // Change resolution to trigger encoder reinitialization.
2007 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2008 WaitForEncodedFrame(2);
2009 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2010 WaitForEncodedFrame(3);
2011
2012 // Ensure the target bitrate is capped by the max bitrate.
2013 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2014 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2015 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2016 .simulcastStream[0]
2017 .targetBitrate *
2018 1000,
2019 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2020
2021 video_stream_encoder_->Stop();
2022}
2023
Åsa Perssona7e34d32021-01-20 15:36:13 +01002024TEST_F(VideoStreamEncoderTest,
2025 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2026 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2027 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2028 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2029 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2030 fake_encoder_.SetResolutionBitrateLimits(
2031 {kEncoderLimits270p, kEncoderLimits360p});
2032
2033 // Two streams, highest stream active.
2034 VideoEncoderConfig config;
2035 const int kNumStreams = 2;
2036 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2037 config.max_bitrate_bps = 0;
2038 config.simulcast_layers[0].active = false;
2039 config.simulcast_layers[1].active = true;
2040 config.video_stream_factory =
2041 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2042 "VP8", /*max qp*/ 56, /*screencast*/ false,
2043 /*screenshare enabled*/ false);
2044 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2045
2046 // The encoder bitrate limits for 270p should be used.
2047 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2048 EXPECT_FALSE(WaitForFrame(1000));
2049 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2050 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2051 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2052 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2053 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2054
2055 // The encoder bitrate limits for 360p should be used.
2056 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2057 EXPECT_FALSE(WaitForFrame(1000));
2058 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2059 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2060 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2061 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2062
2063 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2064 video_source_.IncomingCapturedFrame(
2065 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2066 EXPECT_FALSE(WaitForFrame(1000));
2067 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2068 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2069 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2070 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2071
2072 // Resolution higher than 360p. Encoder limits should be ignored.
2073 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2074 EXPECT_FALSE(WaitForFrame(1000));
2075 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2076 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2077 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2078 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2079 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2080 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2081 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2082 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2083
2084 // Resolution lower than 270p. The encoder limits for 270p should be used.
2085 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2086 EXPECT_FALSE(WaitForFrame(1000));
2087 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2088 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2089 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2090 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2091
2092 video_stream_encoder_->Stop();
2093}
2094
2095TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002096 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2097 // Two streams, highest stream active.
2098 VideoEncoderConfig config;
2099 const int kNumStreams = 2;
2100 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2101 config.max_bitrate_bps = 0;
2102 config.simulcast_layers[0].active = false;
2103 config.simulcast_layers[1].active = true;
2104 config.video_stream_factory =
2105 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2106 "VP8", /*max qp*/ 56, /*screencast*/ false,
2107 /*screenshare enabled*/ false);
2108 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2109
2110 // Default bitrate limits for 270p should be used.
2111 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2112 kDefaultLimits270p =
2113 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002114 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002115 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2116 EXPECT_FALSE(WaitForFrame(1000));
2117 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2118 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2119 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2120 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2121 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2122
2123 // Default bitrate limits for 360p should be used.
2124 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2125 kDefaultLimits360p =
2126 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002127 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002128 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2129 EXPECT_FALSE(WaitForFrame(1000));
2130 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2131 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2132 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2133 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2134
2135 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2136 video_source_.IncomingCapturedFrame(
2137 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2138 EXPECT_FALSE(WaitForFrame(1000));
2139 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2140 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2141 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2142 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2143
2144 // Default bitrate limits for 540p should be used.
2145 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2146 kDefaultLimits540p =
2147 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002148 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002149 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2150 EXPECT_FALSE(WaitForFrame(1000));
2151 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2152 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2153 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2154 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2155
2156 video_stream_encoder_->Stop();
2157}
2158
2159TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002160 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2161 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2162 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2163 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2164 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2165 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2166 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2167 fake_encoder_.SetResolutionBitrateLimits(
2168 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2169
2170 // Three streams, middle stream active.
2171 VideoEncoderConfig config;
2172 const int kNumStreams = 3;
2173 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2174 config.simulcast_layers[0].active = false;
2175 config.simulcast_layers[1].active = true;
2176 config.simulcast_layers[2].active = false;
2177 config.video_stream_factory =
2178 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2179 "VP8", /*max qp*/ 56, /*screencast*/ false,
2180 /*screenshare enabled*/ false);
2181 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2182
2183 // The encoder bitrate limits for 360p should be used.
2184 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2185 EXPECT_FALSE(WaitForFrame(1000));
2186 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2187 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2188 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2189 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2190 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2191
2192 // The encoder bitrate limits for 270p should be used.
2193 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2194 EXPECT_FALSE(WaitForFrame(1000));
2195 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2196 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2197 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2198 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2199
2200 video_stream_encoder_->Stop();
2201}
2202
2203TEST_F(VideoStreamEncoderTest,
2204 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2205 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2206 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2207 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2208 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2209 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2210 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2211 fake_encoder_.SetResolutionBitrateLimits(
2212 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2213
2214 // Three streams, lowest stream active.
2215 VideoEncoderConfig config;
2216 const int kNumStreams = 3;
2217 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2218 config.simulcast_layers[0].active = true;
2219 config.simulcast_layers[1].active = false;
2220 config.simulcast_layers[2].active = false;
2221 config.video_stream_factory =
2222 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2223 "VP8", /*max qp*/ 56, /*screencast*/ false,
2224 /*screenshare enabled*/ false);
2225 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2226
2227 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2228 // on lowest stream, limits for 270p should not be used
2229 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2230 EXPECT_FALSE(WaitForFrame(1000));
2231 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2232 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2233 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2234 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2235 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2236
2237 video_stream_encoder_->Stop();
2238}
2239
2240TEST_F(VideoStreamEncoderTest,
2241 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2242 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2243 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2244 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2245 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2246 fake_encoder_.SetResolutionBitrateLimits(
2247 {kEncoderLimits270p, kEncoderLimits360p});
2248 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2249
2250 // Two streams, highest stream active.
2251 VideoEncoderConfig config;
2252 const int kNumStreams = 2;
2253 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2254 config.simulcast_layers[0].active = false;
2255 config.simulcast_layers[1].active = true;
2256 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2257 config.video_stream_factory =
2258 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2259 "VP8", /*max qp*/ 56, /*screencast*/ false,
2260 /*screenshare enabled*/ false);
2261 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2262
2263 // The encoder bitrate limits for 270p should be used.
2264 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2265 EXPECT_FALSE(WaitForFrame(1000));
2266 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2267 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2268 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2269 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2270 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2271
2272 // The max configured bitrate is less than the encoder limit for 360p.
2273 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2274 EXPECT_FALSE(WaitForFrame(1000));
2275 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2276 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2277 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2278 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2279
2280 video_stream_encoder_->Stop();
2281}
2282
mflodmancc3d4422017-08-03 08:27:51 -07002283TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002284 EXPECT_TRUE(video_source_.has_sinks());
2285 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002286 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002287 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002288 EXPECT_FALSE(video_source_.has_sinks());
2289 EXPECT_TRUE(new_video_source.has_sinks());
2290
mflodmancc3d4422017-08-03 08:27:51 -07002291 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002292}
2293
mflodmancc3d4422017-08-03 08:27:51 -07002294TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002295 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002296 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002297 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002298 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002299}
2300
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002301class ResolutionAlignmentTest
2302 : public VideoStreamEncoderTest,
2303 public ::testing::WithParamInterface<
2304 ::testing::tuple<int, std::vector<double>>> {
2305 public:
2306 ResolutionAlignmentTest()
2307 : requested_alignment_(::testing::get<0>(GetParam())),
2308 scale_factors_(::testing::get<1>(GetParam())) {}
2309
2310 protected:
2311 const int requested_alignment_;
2312 const std::vector<double> scale_factors_;
2313};
2314
2315INSTANTIATE_TEST_SUITE_P(
2316 AlignmentAndScaleFactors,
2317 ResolutionAlignmentTest,
2318 ::testing::Combine(
2319 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2320 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2321 std::vector<double>{-1.0, -1.0},
2322 std::vector<double>{-1.0, -1.0, -1.0},
2323 std::vector<double>{4.0, 2.0, 1.0},
2324 std::vector<double>{9999.0, -1.0, 1.0},
2325 std::vector<double>{3.99, 2.01, 1.0},
2326 std::vector<double>{4.9, 1.7, 1.25},
2327 std::vector<double>{10.0, 4.0, 3.0},
2328 std::vector<double>{1.75, 3.5},
2329 std::vector<double>{1.5, 2.5},
2330 std::vector<double>{1.3, 1.0})));
2331
2332TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2333 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002334 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002335 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2336 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2337
2338 // Fill config with the scaling factor by which to reduce encoding size.
2339 const int num_streams = scale_factors_.size();
2340 VideoEncoderConfig config;
2341 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2342 for (int i = 0; i < num_streams; ++i) {
2343 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2344 }
2345 config.video_stream_factory =
2346 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2347 "VP8", /*max qp*/ 56, /*screencast*/ false,
2348 /*screenshare enabled*/ false);
2349 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2350
Henrik Boström381d1092020-05-12 18:49:07 +02002351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002352 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2353 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2354 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2355 // Wait for all layers before triggering event.
2356 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002357
2358 // On the 1st frame, we should have initialized the encoder and
2359 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002360 int64_t timestamp_ms = kFrameIntervalMs;
2361 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2362 WaitForEncodedFrame(timestamp_ms);
2363 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002364
2365 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2366 // (It's up the to the encoder to potentially drop the previous frame,
2367 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002368 timestamp_ms += kFrameIntervalMs;
2369 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2370 WaitForEncodedFrame(timestamp_ms);
2371 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2372
2373 VideoCodec codec = fake_encoder_.video_codec();
2374 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2375 // Frame size should be a multiple of the requested alignment.
2376 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2377 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2378 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2379 // Aspect ratio should match.
2380 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2381 codec.height * codec.simulcastStream[i].width);
2382 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002383
2384 video_stream_encoder_->Stop();
2385}
2386
Jonathan Yubc771b72017-12-08 17:04:29 -08002387TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2388 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002389 const int kWidth = 1280;
2390 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002391
2392 // We rely on the automatic resolution adaptation, but we handle framerate
2393 // adaptation manually by mocking the stats proxy.
2394 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002395
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002396 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002398 DataRate::BitsPerSec(kTargetBitrateBps),
2399 DataRate::BitsPerSec(kTargetBitrateBps),
2400 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002401 video_stream_encoder_->SetSource(&video_source_,
2402 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002403 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002404 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002405 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002406 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2407
Jonathan Yubc771b72017-12-08 17:04:29 -08002408 // Adapt down as far as possible.
2409 rtc::VideoSinkWants last_wants;
2410 int64_t t = 1;
2411 int loop_count = 0;
2412 do {
2413 ++loop_count;
2414 last_wants = video_source_.sink_wants();
2415
2416 // Simulate the framerate we've been asked to adapt to.
2417 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2418 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2419 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2420 mock_stats.input_frame_rate = fps;
2421 stats_proxy_->SetMockStats(mock_stats);
2422
2423 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2424 sink_.WaitForEncodedFrame(t);
2425 t += frame_interval_ms;
2426
mflodmancc3d4422017-08-03 08:27:51 -07002427 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002428 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002429 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002430 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2431 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002432 } while (video_source_.sink_wants().max_pixel_count <
2433 last_wants.max_pixel_count ||
2434 video_source_.sink_wants().max_framerate_fps <
2435 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002436
Jonathan Yubc771b72017-12-08 17:04:29 -08002437 // Verify that we've adapted all the way down.
2438 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002439 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002440 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2441 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002442 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002443 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2444 *video_source_.last_sent_height());
2445 EXPECT_EQ(kMinBalancedFramerateFps,
2446 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002447
Jonathan Yubc771b72017-12-08 17:04:29 -08002448 // Adapt back up the same number of times we adapted down.
2449 for (int i = 0; i < loop_count - 1; ++i) {
2450 last_wants = video_source_.sink_wants();
2451
2452 // Simulate the framerate we've been asked to adapt to.
2453 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2454 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2455 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2456 mock_stats.input_frame_rate = fps;
2457 stats_proxy_->SetMockStats(mock_stats);
2458
2459 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2460 sink_.WaitForEncodedFrame(t);
2461 t += frame_interval_ms;
2462
Henrik Boström91aa7322020-04-28 12:24:33 +02002463 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002464 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002465 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002466 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2467 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002468 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2469 last_wants.max_pixel_count ||
2470 video_source_.sink_wants().max_framerate_fps >
2471 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002472 }
2473
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002474 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002475 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002476 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002477 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2478 EXPECT_EQ((loop_count - 1) * 2,
2479 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002480
mflodmancc3d4422017-08-03 08:27:51 -07002481 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002482}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002483
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002484TEST_F(VideoStreamEncoderTest,
2485 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2486 video_stream_encoder_->OnBitrateUpdated(
2487 DataRate::BitsPerSec(kTargetBitrateBps),
2488 DataRate::BitsPerSec(kTargetBitrateBps),
2489 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002490 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002491
2492 const int kFrameWidth = 1280;
2493 const int kFrameHeight = 720;
2494
2495 int64_t ntp_time = kFrameIntervalMs;
2496
2497 // Force an input frame rate to be available, or the adaptation call won't
2498 // know what framerate to adapt form.
2499 const int kInputFps = 30;
2500 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2501 stats.input_frame_rate = kInputFps;
2502 stats_proxy_->SetMockStats(stats);
2503
2504 video_source_.set_adaptation_enabled(true);
2505 video_stream_encoder_->SetSource(
2506 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002507 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002508 video_source_.IncomingCapturedFrame(
2509 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2510 sink_.WaitForEncodedFrame(ntp_time);
2511 ntp_time += kFrameIntervalMs;
2512
2513 // Trigger CPU overuse.
2514 video_stream_encoder_->TriggerCpuOveruse();
2515 video_source_.IncomingCapturedFrame(
2516 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2517 sink_.WaitForEncodedFrame(ntp_time);
2518 ntp_time += kFrameIntervalMs;
2519
2520 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2521 EXPECT_EQ(std::numeric_limits<int>::max(),
2522 video_source_.sink_wants().max_pixel_count);
2523 // Some framerate constraint should be set.
2524 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2525 EXPECT_LT(restricted_fps, kInputFps);
2526 video_source_.IncomingCapturedFrame(
2527 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2528 sink_.WaitForEncodedFrame(ntp_time);
2529 ntp_time += 100;
2530
Henrik Boström2671dac2020-05-19 16:29:09 +02002531 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002532 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2533 // Give the encoder queue time to process the change in degradation preference
2534 // by waiting for an encoded frame.
2535 video_source_.IncomingCapturedFrame(
2536 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2537 sink_.WaitForEncodedFrame(ntp_time);
2538 ntp_time += kFrameIntervalMs;
2539
2540 video_stream_encoder_->TriggerQualityLow();
2541 video_source_.IncomingCapturedFrame(
2542 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2543 sink_.WaitForEncodedFrame(ntp_time);
2544 ntp_time += kFrameIntervalMs;
2545
2546 // Some resolution constraint should be set.
2547 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2548 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2549 kFrameWidth * kFrameHeight);
2550 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2551
2552 int pixel_count = video_source_.sink_wants().max_pixel_count;
2553 // Triggering a CPU underuse should not change the sink wants since it has
2554 // not been overused for resolution since we changed degradation preference.
2555 video_stream_encoder_->TriggerCpuUnderuse();
2556 video_source_.IncomingCapturedFrame(
2557 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2558 sink_.WaitForEncodedFrame(ntp_time);
2559 ntp_time += kFrameIntervalMs;
2560 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2561 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2562
Evan Shrubsole64469032020-06-11 10:45:29 +02002563 // Change the degradation preference back. CPU underuse should not adapt since
2564 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002565 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002566 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2567 video_source_.IncomingCapturedFrame(
2568 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2569 sink_.WaitForEncodedFrame(ntp_time);
2570 ntp_time += 100;
2571 // Resolution adaptations is gone after changing degradation preference.
2572 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2573 EXPECT_EQ(std::numeric_limits<int>::max(),
2574 video_source_.sink_wants().max_pixel_count);
2575 // The fps adaptation from above is now back.
2576 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2577
2578 // Trigger CPU underuse.
2579 video_stream_encoder_->TriggerCpuUnderuse();
2580 video_source_.IncomingCapturedFrame(
2581 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2582 sink_.WaitForEncodedFrame(ntp_time);
2583 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002584 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2585
2586 // Trigger QP underuse, fps should return to normal.
2587 video_stream_encoder_->TriggerQualityHigh();
2588 video_source_.IncomingCapturedFrame(
2589 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2590 sink_.WaitForEncodedFrame(ntp_time);
2591 ntp_time += kFrameIntervalMs;
2592 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002593
2594 video_stream_encoder_->Stop();
2595}
2596
mflodmancc3d4422017-08-03 08:27:51 -07002597TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002599 DataRate::BitsPerSec(kTargetBitrateBps),
2600 DataRate::BitsPerSec(kTargetBitrateBps),
2601 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002602 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002603
sprangc5d62e22017-04-02 23:53:04 -07002604 const int kFrameWidth = 1280;
2605 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002606
Åsa Persson8c1bf952018-09-13 10:42:19 +02002607 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002608
kthelgason5e13d412016-12-01 03:59:51 -08002609 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002610 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002611 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002612 frame_timestamp += kFrameIntervalMs;
2613
perkj803d97f2016-11-01 11:45:46 -07002614 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002615 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002616 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002617 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002618 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002619 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002620
asapersson0944a802017-04-07 00:57:58 -07002621 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002622 // wanted resolution.
2623 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2624 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2625 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002626 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002627
2628 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002629 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002630 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002631 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002632 // Give the encoder queue time to process the change in degradation preference
2633 // by waiting for an encoded frame.
2634 new_video_source.IncomingCapturedFrame(
2635 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2636 sink_.WaitForEncodedFrame(frame_timestamp);
2637 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002638 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002639 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002640
sprangc5d62e22017-04-02 23:53:04 -07002641 // Force an input frame rate to be available, or the adaptation call won't
2642 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002643 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002644 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002645 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002646 stats_proxy_->SetMockStats(stats);
2647
mflodmancc3d4422017-08-03 08:27:51 -07002648 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002649 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002650 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002651 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002652 frame_timestamp += kFrameIntervalMs;
2653
2654 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002655 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002656 EXPECT_EQ(std::numeric_limits<int>::max(),
2657 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002658 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002659
asapersson02465b82017-04-10 01:12:52 -07002660 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002661 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2662 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002663 // Give the encoder queue time to process the change in degradation preference
2664 // by waiting for an encoded frame.
2665 new_video_source.IncomingCapturedFrame(
2666 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2667 sink_.WaitForEncodedFrame(frame_timestamp);
2668 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002669 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002670
mflodmancc3d4422017-08-03 08:27:51 -07002671 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002672 new_video_source.IncomingCapturedFrame(
2673 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002674 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002675 frame_timestamp += kFrameIntervalMs;
2676
2677 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002678 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002679
2680 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002681 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002682 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002683 // Give the encoder queue time to process the change in degradation preference
2684 // by waiting for an encoded frame.
2685 new_video_source.IncomingCapturedFrame(
2686 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2687 sink_.WaitForEncodedFrame(frame_timestamp);
2688 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002689 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2690 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002691 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002692 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002693
2694 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002695 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002696 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002697 // Give the encoder queue time to process the change in degradation preference
2698 // by waiting for an encoded frame.
2699 new_video_source.IncomingCapturedFrame(
2700 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2701 sink_.WaitForEncodedFrame(frame_timestamp);
2702 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002703 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2704 EXPECT_EQ(std::numeric_limits<int>::max(),
2705 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002706 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002707
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002709}
2710
mflodmancc3d4422017-08-03 08:27:51 -07002711TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002713 DataRate::BitsPerSec(kTargetBitrateBps),
2714 DataRate::BitsPerSec(kTargetBitrateBps),
2715 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002716
asaperssonfab67072017-04-04 05:51:49 -07002717 const int kWidth = 1280;
2718 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002719 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002720 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002721 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2722 EXPECT_FALSE(stats.bw_limited_resolution);
2723 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2724
2725 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002726 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002727 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002728 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002729
2730 stats = stats_proxy_->GetStats();
2731 EXPECT_TRUE(stats.bw_limited_resolution);
2732 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2733
2734 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002735 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002736 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002737 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002738
2739 stats = stats_proxy_->GetStats();
2740 EXPECT_FALSE(stats.bw_limited_resolution);
2741 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2742 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2743
mflodmancc3d4422017-08-03 08:27:51 -07002744 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002745}
2746
mflodmancc3d4422017-08-03 08:27:51 -07002747TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002749 DataRate::BitsPerSec(kTargetBitrateBps),
2750 DataRate::BitsPerSec(kTargetBitrateBps),
2751 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002752
2753 const int kWidth = 1280;
2754 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002755 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002756 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002757 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2758 EXPECT_FALSE(stats.cpu_limited_resolution);
2759 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2760
2761 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002762 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002763 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002764 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002765
2766 stats = stats_proxy_->GetStats();
2767 EXPECT_TRUE(stats.cpu_limited_resolution);
2768 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2769
2770 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002771 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002772 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002773 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002774
2775 stats = stats_proxy_->GetStats();
2776 EXPECT_FALSE(stats.cpu_limited_resolution);
2777 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002778 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002779
mflodmancc3d4422017-08-03 08:27:51 -07002780 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002781}
2782
mflodmancc3d4422017-08-03 08:27:51 -07002783TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002785 DataRate::BitsPerSec(kTargetBitrateBps),
2786 DataRate::BitsPerSec(kTargetBitrateBps),
2787 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002788
asaperssonfab67072017-04-04 05:51:49 -07002789 const int kWidth = 1280;
2790 const int kHeight = 720;
2791 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002792 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002793 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002794 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002795 EXPECT_FALSE(stats.cpu_limited_resolution);
2796 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2797
asaperssonfab67072017-04-04 05:51:49 -07002798 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002799 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002800 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002801 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002802 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002803 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002804 EXPECT_TRUE(stats.cpu_limited_resolution);
2805 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2806
2807 // Set new source with adaptation still enabled.
2808 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002809 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002810 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002811
asaperssonfab67072017-04-04 05:51:49 -07002812 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002814 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002815 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002816 EXPECT_TRUE(stats.cpu_limited_resolution);
2817 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2818
2819 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002820 video_stream_encoder_->SetSource(&new_video_source,
2821 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002822
asaperssonfab67072017-04-04 05:51:49 -07002823 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002824 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002825 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002826 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002827 EXPECT_FALSE(stats.cpu_limited_resolution);
2828 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2829
2830 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002831 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002832 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002833
asaperssonfab67072017-04-04 05:51:49 -07002834 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002835 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002836 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002837 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002838 EXPECT_TRUE(stats.cpu_limited_resolution);
2839 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2840
asaperssonfab67072017-04-04 05:51:49 -07002841 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002842 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002843 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002844 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002845 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002846 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002847 EXPECT_FALSE(stats.cpu_limited_resolution);
2848 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002849 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002850
mflodmancc3d4422017-08-03 08:27:51 -07002851 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002852}
2853
mflodmancc3d4422017-08-03 08:27:51 -07002854TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002856 DataRate::BitsPerSec(kTargetBitrateBps),
2857 DataRate::BitsPerSec(kTargetBitrateBps),
2858 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002859
asaperssonfab67072017-04-04 05:51:49 -07002860 const int kWidth = 1280;
2861 const int kHeight = 720;
2862 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002863 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002864 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002865 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002866 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002867 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002868
2869 // Set new source with adaptation still enabled.
2870 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002871 video_stream_encoder_->SetSource(&new_video_source,
2872 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002873
asaperssonfab67072017-04-04 05:51:49 -07002874 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002875 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002876 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002877 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002878 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002879 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002880
asaperssonfab67072017-04-04 05:51:49 -07002881 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002882 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002883 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002884 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002885 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002886 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002887 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002888 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002889
asaperssonfab67072017-04-04 05:51:49 -07002890 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002891 video_stream_encoder_->SetSource(&new_video_source,
2892 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002893
asaperssonfab67072017-04-04 05:51:49 -07002894 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002895 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002896 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002897 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002898 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002899 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002900
asapersson02465b82017-04-10 01:12:52 -07002901 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002902 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002903 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002904
asaperssonfab67072017-04-04 05:51:49 -07002905 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002906 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002907 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002908 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002909 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002910 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2911 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002912
mflodmancc3d4422017-08-03 08:27:51 -07002913 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002914}
2915
mflodmancc3d4422017-08-03 08:27:51 -07002916TEST_F(VideoStreamEncoderTest,
2917 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002919 DataRate::BitsPerSec(kTargetBitrateBps),
2920 DataRate::BitsPerSec(kTargetBitrateBps),
2921 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002922
2923 const int kWidth = 1280;
2924 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002925 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002926 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002927 video_source_.IncomingCapturedFrame(
2928 CreateFrame(timestamp_ms, kWidth, kHeight));
2929 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002930 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2931 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2932 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2933
2934 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002935 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002936 timestamp_ms += kFrameIntervalMs;
2937 video_source_.IncomingCapturedFrame(
2938 CreateFrame(timestamp_ms, kWidth, kHeight));
2939 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002940 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2941 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2942 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2943
2944 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002945 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002946 timestamp_ms += kFrameIntervalMs;
2947 video_source_.IncomingCapturedFrame(
2948 CreateFrame(timestamp_ms, kWidth, kHeight));
2949 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002950 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2951 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2952 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2953
Niels Möller4db138e2018-04-19 09:04:13 +02002954 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002955 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002956
2957 VideoEncoderConfig video_encoder_config;
2958 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2959 // Make format different, to force recreation of encoder.
2960 video_encoder_config.video_format.parameters["foo"] = "foo";
2961 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002962 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002963 timestamp_ms += kFrameIntervalMs;
2964 video_source_.IncomingCapturedFrame(
2965 CreateFrame(timestamp_ms, kWidth, kHeight));
2966 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002967 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2968 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2969 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2970
mflodmancc3d4422017-08-03 08:27:51 -07002971 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002972}
2973
mflodmancc3d4422017-08-03 08:27:51 -07002974TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002975 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002977 DataRate::BitsPerSec(kTargetBitrateBps),
2978 DataRate::BitsPerSec(kTargetBitrateBps),
2979 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2980
2981 const int kWidth = 1280;
2982 const int kHeight = 720;
2983 int sequence = 1;
2984
2985 // Enable BALANCED preference, no initial limitation.
2986 test::FrameForwarder source;
2987 video_stream_encoder_->SetSource(&source,
2988 webrtc::DegradationPreference::BALANCED);
2989 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2990 WaitForEncodedFrame(sequence++);
2991 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2992 EXPECT_FALSE(stats.cpu_limited_resolution);
2993 EXPECT_FALSE(stats.cpu_limited_framerate);
2994 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2995
2996 // Trigger CPU overuse, should now adapt down.
2997 video_stream_encoder_->TriggerCpuOveruse();
2998 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2999 WaitForEncodedFrame(sequence++);
3000 stats = stats_proxy_->GetStats();
3001 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3002
3003 // Set new degradation preference should clear restrictions since we changed
3004 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003005 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003006 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3007 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3008 WaitForEncodedFrame(sequence++);
3009 stats = stats_proxy_->GetStats();
3010 EXPECT_FALSE(stats.cpu_limited_resolution);
3011 EXPECT_FALSE(stats.cpu_limited_framerate);
3012 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3013
3014 // Force an input frame rate to be available, or the adaptation call won't
3015 // know what framerate to adapt from.
3016 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3017 mock_stats.input_frame_rate = 30;
3018 stats_proxy_->SetMockStats(mock_stats);
3019 video_stream_encoder_->TriggerCpuOveruse();
3020 stats_proxy_->ResetMockStats();
3021 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3022 WaitForEncodedFrame(sequence++);
3023
3024 // We have now adapted once.
3025 stats = stats_proxy_->GetStats();
3026 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3027
3028 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003029 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3030 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003031 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3032 WaitForEncodedFrame(sequence++);
3033 stats = stats_proxy_->GetStats();
3034 EXPECT_FALSE(stats.cpu_limited_resolution);
3035 EXPECT_FALSE(stats.cpu_limited_framerate);
3036 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3037
3038 video_stream_encoder_->Stop();
3039}
3040
3041TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003042 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003044 DataRate::BitsPerSec(kTargetBitrateBps),
3045 DataRate::BitsPerSec(kTargetBitrateBps),
3046 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003047
asapersson0944a802017-04-07 00:57:58 -07003048 const int kWidth = 1280;
3049 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003050 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003051
asaperssonfab67072017-04-04 05:51:49 -07003052 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003053 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003054 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003055 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003056 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003057 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3058
asapersson02465b82017-04-10 01:12:52 -07003059 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003060 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003061 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003062 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003063 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003064 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003065 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003066 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3067
3068 // Set new source with adaptation still enabled.
3069 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003070 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003071 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003072
3073 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003074 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003075 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003076 stats = stats_proxy_->GetStats();
3077 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003078 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003079 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3080
sprangc5d62e22017-04-02 23:53:04 -07003081 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003082 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003083 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003084 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003085 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003086 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003087 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003088 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003089 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003090 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003091 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3092
sprangc5d62e22017-04-02 23:53:04 -07003093 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003094 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003095 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3096 mock_stats.input_frame_rate = 30;
3097 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003098 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003099 stats_proxy_->ResetMockStats();
3100
3101 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003102 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003103 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003104
3105 // Framerate now adapted.
3106 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003107 EXPECT_FALSE(stats.cpu_limited_resolution);
3108 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003109 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3110
3111 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003112 video_stream_encoder_->SetSource(&new_video_source,
3113 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003114 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003115 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003116 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003117
3118 stats = stats_proxy_->GetStats();
3119 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003120 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003121 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3122
3123 // Try to trigger overuse. Should not succeed.
3124 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003125 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003126 stats_proxy_->ResetMockStats();
3127
3128 stats = stats_proxy_->GetStats();
3129 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003130 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003131 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3132
3133 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003134 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003135 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003136 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003137 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003138 stats = stats_proxy_->GetStats();
3139 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003140 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003141 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003142
3143 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003144 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003145 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003146 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003147 stats = stats_proxy_->GetStats();
3148 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003149 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003150 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3151
3152 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003154 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003155 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003156 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003157 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003158 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003159 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003160 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003161 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003162 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3163
3164 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003165 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003166 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003167 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003168 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003169 stats = stats_proxy_->GetStats();
3170 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003171 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003172 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003173 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003174
mflodmancc3d4422017-08-03 08:27:51 -07003175 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003176}
3177
mflodmancc3d4422017-08-03 08:27:51 -07003178TEST_F(VideoStreamEncoderTest,
3179 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003180 const int kWidth = 1280;
3181 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003182 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003183 DataRate::BitsPerSec(kTargetBitrateBps),
3184 DataRate::BitsPerSec(kTargetBitrateBps),
3185 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003186
asaperssonfab67072017-04-04 05:51:49 -07003187 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003188 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003189
asaperssonfab67072017-04-04 05:51:49 -07003190 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003191 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003192
asaperssonfab67072017-04-04 05:51:49 -07003193 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003194 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003195
asaperssonfab67072017-04-04 05:51:49 -07003196 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003197 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003198
kthelgason876222f2016-11-29 01:44:11 -08003199 // Expect a scale down.
3200 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003201 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003202
asapersson02465b82017-04-10 01:12:52 -07003203 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003204 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003205 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003206 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003207
asaperssonfab67072017-04-04 05:51:49 -07003208 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003209 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003210 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003211 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003212
asaperssonfab67072017-04-04 05:51:49 -07003213 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003214 EXPECT_EQ(std::numeric_limits<int>::max(),
3215 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003216
asaperssonfab67072017-04-04 05:51:49 -07003217 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003218 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003219 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003220 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003221
asapersson02465b82017-04-10 01:12:52 -07003222 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003223 EXPECT_EQ(std::numeric_limits<int>::max(),
3224 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003225
mflodmancc3d4422017-08-03 08:27:51 -07003226 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003227}
3228
mflodmancc3d4422017-08-03 08:27:51 -07003229TEST_F(VideoStreamEncoderTest,
3230 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003231 const int kWidth = 1280;
3232 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003233 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003234 DataRate::BitsPerSec(kTargetBitrateBps),
3235 DataRate::BitsPerSec(kTargetBitrateBps),
3236 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003237
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003238 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003239 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003240 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003241 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003242
3243 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003244 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003245 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003246 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3247 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3248
3249 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003250 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003251 EXPECT_THAT(source.sink_wants(),
3252 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003253 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3254 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3255 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3256
3257 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003258 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003259 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3260 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3261 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3262
mflodmancc3d4422017-08-03 08:27:51 -07003263 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003264}
3265
mflodmancc3d4422017-08-03 08:27:51 -07003266TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003267 const int kWidth = 1280;
3268 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003269 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003270 DataRate::BitsPerSec(kTargetBitrateBps),
3271 DataRate::BitsPerSec(kTargetBitrateBps),
3272 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003273
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003274 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003275 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003276 video_stream_encoder_->SetSource(&source,
3277 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003278 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3279 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003280 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003281
3282 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003283 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003284 EXPECT_THAT(source.sink_wants(),
3285 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3287 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3288 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3289
3290 // Trigger adapt down for same input resolution, expect no change.
3291 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3292 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003293 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003294 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3295 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3296 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3297
3298 // Trigger adapt down for larger input resolution, expect no change.
3299 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3300 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003301 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003302 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3303 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3304 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3305
mflodmancc3d4422017-08-03 08:27:51 -07003306 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003307}
3308
mflodmancc3d4422017-08-03 08:27:51 -07003309TEST_F(VideoStreamEncoderTest,
3310 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003311 const int kWidth = 1280;
3312 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003314 DataRate::BitsPerSec(kTargetBitrateBps),
3315 DataRate::BitsPerSec(kTargetBitrateBps),
3316 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003317
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003318 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003319 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003320 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003321 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003322
3323 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003324 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003325 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003326 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3327 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3328
3329 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003330 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003331 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003332 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3333 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3334
mflodmancc3d4422017-08-03 08:27:51 -07003335 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003336}
3337
mflodmancc3d4422017-08-03 08:27:51 -07003338TEST_F(VideoStreamEncoderTest,
3339 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003340 const int kWidth = 1280;
3341 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003343 DataRate::BitsPerSec(kTargetBitrateBps),
3344 DataRate::BitsPerSec(kTargetBitrateBps),
3345 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003346
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003347 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003348 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003349 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003350 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003351
3352 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003353 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003354 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003355 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003356 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3357
3358 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003359 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003360 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003361 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003362 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3363
mflodmancc3d4422017-08-03 08:27:51 -07003364 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003365}
3366
mflodmancc3d4422017-08-03 08:27:51 -07003367TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003368 const int kWidth = 1280;
3369 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003371 DataRate::BitsPerSec(kTargetBitrateBps),
3372 DataRate::BitsPerSec(kTargetBitrateBps),
3373 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003374
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003375 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003376 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003377 video_stream_encoder_->SetSource(&source,
3378 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003379
3380 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3381 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003382 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003383 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3384 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3385 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3386
3387 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003388 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003389 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3391 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3392 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3393
mflodmancc3d4422017-08-03 08:27:51 -07003394 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003395}
3396
mflodmancc3d4422017-08-03 08:27:51 -07003397TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003398 const int kWidth = 1280;
3399 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003401 DataRate::BitsPerSec(kTargetBitrateBps),
3402 DataRate::BitsPerSec(kTargetBitrateBps),
3403 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003404
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003405 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003406 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003407 video_stream_encoder_->SetSource(&source,
3408 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003409
3410 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3411 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003412 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3414 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3415 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3416
3417 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003418 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003420 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3422 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
mflodmancc3d4422017-08-03 08:27:51 -07003424 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003425}
3426
mflodmancc3d4422017-08-03 08:27:51 -07003427TEST_F(VideoStreamEncoderTest,
3428 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003429 const int kWidth = 1280;
3430 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003431 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003432 DataRate::BitsPerSec(kTargetBitrateBps),
3433 DataRate::BitsPerSec(kTargetBitrateBps),
3434 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003435
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003436 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003437 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003438 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003439 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003440 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003441
3442 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003443 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003444 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003445 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3446 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3447
3448 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003449 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003450 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003451 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003452 EXPECT_THAT(source.sink_wants(),
3453 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3455 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3456
3457 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003458 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003459 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003460 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3461 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3462 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3463
mflodmancc3d4422017-08-03 08:27:51 -07003464 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003465}
3466
mflodmancc3d4422017-08-03 08:27:51 -07003467TEST_F(VideoStreamEncoderTest,
3468 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003469 const int kWidth = 1280;
3470 const int kHeight = 720;
3471 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003473 DataRate::BitsPerSec(kTargetBitrateBps),
3474 DataRate::BitsPerSec(kTargetBitrateBps),
3475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003476
3477 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3478 stats.input_frame_rate = kInputFps;
3479 stats_proxy_->SetMockStats(stats);
3480
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003481 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003482 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3483 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003484 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003485
3486 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003487 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003488 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3489 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003490 EXPECT_THAT(video_source_.sink_wants(),
3491 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003492
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003493 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003494 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003495 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003496 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003497 // Give the encoder queue time to process the change in degradation preference
3498 // by waiting for an encoded frame.
3499 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3500 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003501 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003502
3503 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003504 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003505 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3506 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003507 EXPECT_THAT(new_video_source.sink_wants(),
3508 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003509
3510 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003511 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003512 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003513
mflodmancc3d4422017-08-03 08:27:51 -07003514 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003515}
3516
mflodmancc3d4422017-08-03 08:27:51 -07003517TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003518 const int kWidth = 1280;
3519 const int kHeight = 720;
3520 const size_t kNumFrames = 10;
3521
Henrik Boström381d1092020-05-12 18:49:07 +02003522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003523 DataRate::BitsPerSec(kTargetBitrateBps),
3524 DataRate::BitsPerSec(kTargetBitrateBps),
3525 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003526
asaperssond0de2952017-04-21 01:47:31 -07003527 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003528 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003529 video_source_.set_adaptation_enabled(true);
3530
3531 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3532 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3533
3534 int downscales = 0;
3535 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003536 video_source_.IncomingCapturedFrame(
3537 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3538 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003539
asaperssonfab67072017-04-04 05:51:49 -07003540 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003541 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003542 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003543 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003544
3545 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3546 ++downscales;
3547
3548 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3549 EXPECT_EQ(downscales,
3550 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3551 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003552 }
mflodmancc3d4422017-08-03 08:27:51 -07003553 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003554}
3555
mflodmancc3d4422017-08-03 08:27:51 -07003556TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003557 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3558 const int kWidth = 1280;
3559 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003561 DataRate::BitsPerSec(kTargetBitrateBps),
3562 DataRate::BitsPerSec(kTargetBitrateBps),
3563 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003564
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003565 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003566 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003567 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003568 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003569 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003570
Åsa Persson8c1bf952018-09-13 10:42:19 +02003571 int64_t timestamp_ms = kFrameIntervalMs;
3572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003573 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003574 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003575 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3576 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3577
3578 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003579 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003580 timestamp_ms += kFrameIntervalMs;
3581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3582 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003583 EXPECT_THAT(source.sink_wants(),
3584 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003585 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3586 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3587
3588 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003589 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003592 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003593 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003594 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3595 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3596
3597 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003598 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003599 timestamp_ms += kFrameIntervalMs;
3600 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3601 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003602 EXPECT_THAT(source.sink_wants(),
3603 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003604 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3605 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3606
3607 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003608 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003609 timestamp_ms += kFrameIntervalMs;
3610 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003611 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003612 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003613 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3614 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3615
mflodmancc3d4422017-08-03 08:27:51 -07003616 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003617}
3618
mflodmancc3d4422017-08-03 08:27:51 -07003619TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003620 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3621 const int kWidth = 1280;
3622 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003624 DataRate::BitsPerSec(kTargetBitrateBps),
3625 DataRate::BitsPerSec(kTargetBitrateBps),
3626 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003627
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003628 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003629 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003630 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003631 video_stream_encoder_->SetSource(&source,
3632 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003633
Åsa Persson8c1bf952018-09-13 10:42:19 +02003634 int64_t timestamp_ms = kFrameIntervalMs;
3635 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003636 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003637 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003638 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3639 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3640
3641 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003642 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003643 timestamp_ms += kFrameIntervalMs;
3644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3645 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003646 EXPECT_THAT(source.sink_wants(),
3647 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3649 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3650
3651 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003652 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003653 timestamp_ms += kFrameIntervalMs;
3654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003655 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003656 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3658 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3659
3660 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003661 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003662 timestamp_ms += kFrameIntervalMs;
3663 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3664 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003665 EXPECT_THAT(source.sink_wants(),
3666 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003667 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3668 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3669
3670 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003671 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003672 timestamp_ms += kFrameIntervalMs;
3673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003674 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3677 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3678
mflodmancc3d4422017-08-03 08:27:51 -07003679 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003680}
3681
Sergey Silkin41c650b2019-10-14 13:12:19 +02003682TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3683 fake_encoder_.SetResolutionBitrateLimits(
3684 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3685
Henrik Boström381d1092020-05-12 18:49:07 +02003686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003687 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3688 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3689 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3690 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003691
3692 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003693 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003694 source.set_adaptation_enabled(true);
3695 video_stream_encoder_->SetSource(
3696 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3697
3698 // Insert 720p frame.
3699 int64_t timestamp_ms = kFrameIntervalMs;
3700 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3701 WaitForEncodedFrame(1280, 720);
3702
3703 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003705 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3706 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3707 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3708 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003709 video_stream_encoder_->TriggerQualityLow();
3710
3711 // Insert 720p frame. It should be downscaled and encoded.
3712 timestamp_ms += kFrameIntervalMs;
3713 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3714 WaitForEncodedFrame(960, 540);
3715
3716 // Trigger adapt up. Higher resolution should not be requested duo to lack
3717 // of bitrate.
3718 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003719 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003720
3721 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003722 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003723 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3724 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3725 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3726 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003727
3728 // Trigger adapt up. Higher resolution should be requested.
3729 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003730 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003731
3732 video_stream_encoder_->Stop();
3733}
3734
3735TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3736 fake_encoder_.SetResolutionBitrateLimits(
3737 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3738
3739 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003741 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3742 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3743 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3744 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003745
3746 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003747 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003748 source.set_adaptation_enabled(true);
3749 video_stream_encoder_->SetSource(
3750 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3751
3752 // Insert 720p frame. It should be dropped and lower resolution should be
3753 // requested.
3754 int64_t timestamp_ms = kFrameIntervalMs;
3755 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3756 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003757 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003758
3759 // Insert 720p frame. It should be downscaled and encoded.
3760 timestamp_ms += kFrameIntervalMs;
3761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3762 WaitForEncodedFrame(960, 540);
3763
3764 video_stream_encoder_->Stop();
3765}
3766
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003767class BalancedDegradationTest : public VideoStreamEncoderTest {
3768 protected:
3769 void SetupTest() {
3770 // Reset encoder for field trials to take effect.
3771 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003772 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003773
3774 // Enable BALANCED preference.
3775 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003776 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3777 }
3778
3779 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003781 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3782 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003783 }
3784
Åsa Persson45b176f2019-09-30 11:19:05 +02003785 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003786 timestamp_ms_ += kFrameIntervalMs;
3787 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003788 }
3789
3790 void InsertFrameAndWaitForEncoded() {
3791 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003792 sink_.WaitForEncodedFrame(timestamp_ms_);
3793 }
3794
3795 const int kWidth = 640; // pixels:640x360=230400
3796 const int kHeight = 360;
3797 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3798 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003799 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003800};
3801
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003802TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003803 test::ScopedFieldTrials field_trials(
3804 "WebRTC-Video-BalancedDegradationSettings/"
3805 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3806 SetupTest();
3807
3808 // Force input frame rate.
3809 const int kInputFps = 24;
3810 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3811 stats.input_frame_rate = kInputFps;
3812 stats_proxy_->SetMockStats(stats);
3813
Åsa Persson45b176f2019-09-30 11:19:05 +02003814 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003815 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003816
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003817 // Trigger adapt down, expect scaled down framerate and resolution,
3818 // since Fps diff (input-requested:0) < threshold.
3819 video_stream_encoder_->TriggerQualityLow();
3820 EXPECT_THAT(source_.sink_wants(),
3821 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003822
3823 video_stream_encoder_->Stop();
3824}
3825
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003826TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003827 test::ScopedFieldTrials field_trials(
3828 "WebRTC-Video-BalancedDegradationSettings/"
3829 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3830 SetupTest();
3831
3832 // Force input frame rate.
3833 const int kInputFps = 25;
3834 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3835 stats.input_frame_rate = kInputFps;
3836 stats_proxy_->SetMockStats(stats);
3837
Åsa Persson45b176f2019-09-30 11:19:05 +02003838 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003839 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003840
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003841 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3842 // Fps diff (input-requested:1) == threshold.
3843 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003844 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003845
3846 video_stream_encoder_->Stop();
3847}
3848
3849TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3850 test::ScopedFieldTrials field_trials(
3851 "WebRTC-Video-BalancedDegradationSettings/"
3852 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3853 SetupTest();
3854
3855 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3856
Åsa Persson45b176f2019-09-30 11:19:05 +02003857 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003858 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003859
3860 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3861 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003862 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003863
3864 video_stream_encoder_->Stop();
3865}
3866
Åsa Perssonccfb3402019-09-25 15:13:04 +02003867TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003868 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003869 "WebRTC-Video-BalancedDegradationSettings/"
3870 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003871 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003872
Åsa Persson1b247f12019-08-14 17:26:39 +02003873 const int kMinBitrateBps = 425000;
3874 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003875 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003876
Åsa Persson45b176f2019-09-30 11:19:05 +02003877 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003878 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003879 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3880
3881 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3882 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003883 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003884 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003885 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3886
3887 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3888 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003889 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003890 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003891 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3892
Åsa Persson30ab0152019-08-27 12:22:33 +02003893 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3894 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003895 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003896 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003897 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003898 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3899
3900 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003901 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003902 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003903 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003904
Åsa Persson30ab0152019-08-27 12:22:33 +02003905 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003906 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003907 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003908 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003909 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003910 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3911
3912 video_stream_encoder_->Stop();
3913}
3914
Åsa Perssonccfb3402019-09-25 15:13:04 +02003915TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003916 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3917 test::ScopedFieldTrials field_trials(
3918 "WebRTC-Video-BalancedDegradationSettings/"
3919 "pixels:57600|129600|230400,fps:7|24|24/");
3920 SetupTest();
3921 OnBitrateUpdated(kLowTargetBitrateBps);
3922
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003923 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003924
3925 // Insert frame, expect scaled down:
3926 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3927 InsertFrame();
3928 EXPECT_FALSE(WaitForFrame(1000));
3929 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3930 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3931
3932 // Insert frame, expect scaled down:
3933 // resolution (320x180@24fps).
3934 InsertFrame();
3935 EXPECT_FALSE(WaitForFrame(1000));
3936 EXPECT_LT(source_.sink_wants().max_pixel_count,
3937 source_.last_wants().max_pixel_count);
3938 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3939
3940 // Frame should not be dropped (min pixels per frame reached).
3941 InsertFrameAndWaitForEncoded();
3942
3943 video_stream_encoder_->Stop();
3944}
3945
3946TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003947 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003948 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003949 "WebRTC-Video-BalancedDegradationSettings/"
3950 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003951 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003952
Åsa Persson30ab0152019-08-27 12:22:33 +02003953 const int kResolutionMinBitrateBps = 435000;
3954 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003955 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003956
Åsa Persson45b176f2019-09-30 11:19:05 +02003957 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003958 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003959 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3960
3961 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3962 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003963 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003964 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003965 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3966
3967 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3968 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003969 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003970 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003971 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3972
3973 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3974 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003975 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003976 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003977 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3978
Åsa Persson30ab0152019-08-27 12:22:33 +02003979 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3980 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003981 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003982 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003983 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3984
3985 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3986 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003987 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003988 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3989
3990 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003991 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003992 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003993 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003994 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003995 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3996
3997 video_stream_encoder_->Stop();
3998}
3999
Åsa Perssonccfb3402019-09-25 15:13:04 +02004000TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004001 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004002 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004003 "WebRTC-Video-BalancedDegradationSettings/"
4004 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004005 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004006
Åsa Persson30ab0152019-08-27 12:22:33 +02004007 const int kMinBitrateBps = 425000;
4008 const int kTooLowMinBitrateBps = 424000;
4009 const int kResolutionMinBitrateBps = 435000;
4010 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004011 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004012
Åsa Persson45b176f2019-09-30 11:19:05 +02004013 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004014 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004015 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4016
4017 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4018 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004019 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004020 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004021 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4022
4023 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4024 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004025 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004026 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004027 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4028
4029 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4030 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004031 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004032 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004033 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4034
4035 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4036 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004037 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004038 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4039
4040 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004041 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004042 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004043 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004044 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004045 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4046
4047 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004048 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004049 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004050 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004051 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4052
4053 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004054 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004055 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004056 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004057 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004058 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4059
Åsa Persson1b247f12019-08-14 17:26:39 +02004060 video_stream_encoder_->Stop();
4061}
4062
mflodmancc3d4422017-08-03 08:27:51 -07004063TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004064 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4065 const int kWidth = 1280;
4066 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004068 DataRate::BitsPerSec(kTargetBitrateBps),
4069 DataRate::BitsPerSec(kTargetBitrateBps),
4070 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004071
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004072 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004073 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004074 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004075 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004076 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004077
Åsa Persson8c1bf952018-09-13 10:42:19 +02004078 int64_t timestamp_ms = kFrameIntervalMs;
4079 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004080 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004081 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004082 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4083 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4084 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4085 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4086
4087 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004088 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004089 timestamp_ms += kFrameIntervalMs;
4090 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4091 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004092 EXPECT_THAT(source.sink_wants(),
4093 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004094 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4095 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4096 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4097 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4098
4099 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004100 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004101 timestamp_ms += kFrameIntervalMs;
4102 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4103 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004104 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004105 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4106 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4107 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4108 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4109
Jonathan Yubc771b72017-12-08 17:04:29 -08004110 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004111 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004112 timestamp_ms += kFrameIntervalMs;
4113 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4114 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004115 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004116 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4117 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004118 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004119 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4120
Jonathan Yubc771b72017-12-08 17:04:29 -08004121 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004122 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004123 timestamp_ms += kFrameIntervalMs;
4124 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4125 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004126 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004127 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004128 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4129 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4130 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4131 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4132
Jonathan Yubc771b72017-12-08 17:04:29 -08004133 // Trigger quality adapt down, expect no change (min resolution reached).
4134 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004135 timestamp_ms += kFrameIntervalMs;
4136 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4137 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004138 EXPECT_THAT(source.sink_wants(), FpsMax());
4139 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004140 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4141 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4142 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4143 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4144
Evan Shrubsole64469032020-06-11 10:45:29 +02004145 // Trigger quality adapt up, expect upscaled resolution (480x270).
4146 video_stream_encoder_->TriggerQualityHigh();
4147 timestamp_ms += kFrameIntervalMs;
4148 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4149 WaitForEncodedFrame(timestamp_ms);
4150 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4151 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4152 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4153 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4154 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4155
4156 // Trigger quality and cpu adapt up since both are most limited, expect
4157 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004158 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004159 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004160 timestamp_ms += kFrameIntervalMs;
4161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4162 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004163 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004164 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4165 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4166 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004167 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004168
Evan Shrubsole64469032020-06-11 10:45:29 +02004169 // Trigger quality and cpu adapt up since both are most limited, expect
4170 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004171 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004172 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004173 timestamp_ms += kFrameIntervalMs;
4174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4175 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004176 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004177 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004178 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004179 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004180 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4181 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004182
Evan Shrubsole64469032020-06-11 10:45:29 +02004183 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4184 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004185 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004186 timestamp_ms += kFrameIntervalMs;
4187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4188 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004189 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004190 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4191 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004192 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004193 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004194
4195 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004196 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004197 timestamp_ms += kFrameIntervalMs;
4198 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004199 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004200 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004201 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004202 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4203 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004204 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004205 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004206
mflodmancc3d4422017-08-03 08:27:51 -07004207 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004208}
4209
mflodmancc3d4422017-08-03 08:27:51 -07004210TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004211 const int kWidth = 640;
4212 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004213
Henrik Boström381d1092020-05-12 18:49:07 +02004214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004215 DataRate::BitsPerSec(kTargetBitrateBps),
4216 DataRate::BitsPerSec(kTargetBitrateBps),
4217 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004218
perkj803d97f2016-11-01 11:45:46 -07004219 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004220 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004221 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004222 }
4223
mflodmancc3d4422017-08-03 08:27:51 -07004224 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004225 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004226 video_source_.IncomingCapturedFrame(CreateFrame(
4227 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004228 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004229 }
4230
mflodmancc3d4422017-08-03 08:27:51 -07004231 video_stream_encoder_->Stop();
4232 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004233 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004234
Ying Wangef3998f2019-12-09 13:06:53 +01004235 EXPECT_METRIC_EQ(
4236 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4237 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004238 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4239}
4240
mflodmancc3d4422017-08-03 08:27:51 -07004241TEST_F(VideoStreamEncoderTest,
4242 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004244 DataRate::BitsPerSec(kTargetBitrateBps),
4245 DataRate::BitsPerSec(kTargetBitrateBps),
4246 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004247 const int kWidth = 640;
4248 const int kHeight = 360;
4249
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004250 video_stream_encoder_->SetSource(&video_source_,
4251 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004252
4253 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4254 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004255 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004256 }
4257
mflodmancc3d4422017-08-03 08:27:51 -07004258 video_stream_encoder_->Stop();
4259 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004260 stats_proxy_.reset();
4261
4262 EXPECT_EQ(0,
4263 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4264}
4265
Per Kjellanderdcef6412020-10-07 15:09:05 +02004266TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4267 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004268 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004269 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004270
4271 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004272 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004273 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004274 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4275 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004276
Henrik Boström381d1092020-05-12 18:49:07 +02004277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004278 DataRate::BitsPerSec(kLowTargetBitrateBps),
4279 DataRate::BitsPerSec(kLowTargetBitrateBps),
4280 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004281
sprang57c2fff2017-01-16 06:24:02 -08004282 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004283 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4284 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004285 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4286 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4287
Erik Språngd7329ca2019-02-21 21:19:53 +01004288 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004289 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004290 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004291
Per Kjellanderdcef6412020-10-07 15:09:05 +02004292 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004293 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004294 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4295 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004296 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004297 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004298
Per Kjellanderdcef6412020-10-07 15:09:05 +02004299 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004300 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004301 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004302 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004303 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4304 WaitForEncodedFrame(CurrentTimeMs());
4305 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004306 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004307 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004308
mflodmancc3d4422017-08-03 08:27:51 -07004309 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004310}
4311
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004312TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004313 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004314 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004315 kVideoLayersAllocation);
4316
4317 const int kDefaultFps = 30;
4318
4319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4320 DataRate::BitsPerSec(kLowTargetBitrateBps),
4321 DataRate::BitsPerSec(kLowTargetBitrateBps),
4322 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4323
4324 video_source_.IncomingCapturedFrame(
4325 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4326 WaitForEncodedFrame(CurrentTimeMs());
4327 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4328 VideoLayersAllocation last_layer_allocation =
4329 sink_.GetLastVideoLayersAllocation();
4330 // kLowTargetBitrateBps is only enough for one spatial layer.
4331 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4332
4333 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004334 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004335 // Check that encoder has been updated too, not just allocation observer.
4336 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4337 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4338
Erik Språng9d69cbe2020-10-22 17:44:42 +02004339 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004340 int number_of_layers_allocation = 1;
4341 const int64_t start_time_ms = CurrentTimeMs();
4342 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4343 video_source_.IncomingCapturedFrame(
4344 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4345 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004346 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4347 number_of_layers_allocation = sink_.number_of_layers_allocations();
4348 VideoLayersAllocation new_allocation =
4349 sink_.GetLastVideoLayersAllocation();
4350 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4351 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4352 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4353 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4354 .target_bitrate_per_temporal_layer,
4355 last_layer_allocation.active_spatial_layers[0]
4356 .target_bitrate_per_temporal_layer);
4357 last_layer_allocation = new_allocation;
4358 }
4359 }
4360 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4361 video_stream_encoder_->Stop();
4362}
4363
4364TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004365 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004366 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4367 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4368 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004369 VideoEncoderConfig video_encoder_config;
4370 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4371 /* num_streams*/ 3, &video_encoder_config);
4372 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4373 video_encoder_config.content_type =
4374 VideoEncoderConfig::ContentType::kRealtimeVideo;
4375 video_encoder_config.encoder_specific_settings =
4376 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4377 VideoEncoder::GetDefaultVp8Settings());
4378 for (auto& layer : video_encoder_config.simulcast_layers) {
4379 layer.num_temporal_layers = 2;
4380 }
4381 // Simulcast layers are used for enabling/disabling streams.
4382 video_encoder_config.simulcast_layers[0].active = true;
4383 video_encoder_config.simulcast_layers[1].active = false;
4384 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004385 ConfigureEncoder(std::move(video_encoder_config),
4386 VideoStreamEncoder::BitrateAllocationCallbackType::
4387 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004388
4389 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4390 DataRate::BitsPerSec(kTargetBitrateBps),
4391 DataRate::BitsPerSec(kTargetBitrateBps),
4392 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4393
4394 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4395 WaitForEncodedFrame(CurrentTimeMs());
4396 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4397 VideoLayersAllocation last_layer_allocation =
4398 sink_.GetLastVideoLayersAllocation();
4399
4400 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4401 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4402 .target_bitrate_per_temporal_layer,
4403 SizeIs(2));
4404 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4405 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4406 video_stream_encoder_->Stop();
4407}
4408
4409TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004410 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004411 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4412 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4413 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004414 VideoEncoderConfig video_encoder_config;
4415 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4416 /* num_streams*/ 3, &video_encoder_config);
4417 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4418 video_encoder_config.content_type =
4419 VideoEncoderConfig::ContentType::kRealtimeVideo;
4420 video_encoder_config.encoder_specific_settings =
4421 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4422 VideoEncoder::GetDefaultVp8Settings());
4423 for (auto& layer : video_encoder_config.simulcast_layers) {
4424 layer.num_temporal_layers = 2;
4425 }
4426 // Simulcast layers are used for enabling/disabling streams.
4427 video_encoder_config.simulcast_layers[0].active = true;
4428 video_encoder_config.simulcast_layers[1].active = false;
4429 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004430 ConfigureEncoder(std::move(video_encoder_config),
4431 VideoStreamEncoder::BitrateAllocationCallbackType::
4432 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004433
4434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4435 DataRate::BitsPerSec(kTargetBitrateBps),
4436 DataRate::BitsPerSec(kTargetBitrateBps),
4437 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4438
4439 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4440 WaitForEncodedFrame(CurrentTimeMs());
4441 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4442 VideoLayersAllocation last_layer_allocation =
4443 sink_.GetLastVideoLayersAllocation();
4444
4445 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4446 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4447 .target_bitrate_per_temporal_layer,
4448 SizeIs(2));
4449 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4450
4451 video_stream_encoder_->Stop();
4452}
4453
4454TEST_F(VideoStreamEncoderTest,
4455 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4456 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4457 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004458 VideoEncoderConfig video_encoder_config;
4459 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4460 /* num_streams*/ 1, &video_encoder_config);
4461 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4462 video_encoder_config.content_type =
4463 VideoEncoderConfig::ContentType::kRealtimeVideo;
4464 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4465 vp9_settings.numberOfSpatialLayers = 2;
4466 vp9_settings.numberOfTemporalLayers = 2;
4467 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4468 vp9_settings.automaticResizeOn = false;
4469 video_encoder_config.encoder_specific_settings =
4470 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4471 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004472 ConfigureEncoder(std::move(video_encoder_config),
4473 VideoStreamEncoder::BitrateAllocationCallbackType::
4474 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004475
4476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4477 DataRate::BitsPerSec(kTargetBitrateBps),
4478 DataRate::BitsPerSec(kTargetBitrateBps),
4479 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4480
4481 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4482 WaitForEncodedFrame(CurrentTimeMs());
4483 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4484 VideoLayersAllocation last_layer_allocation =
4485 sink_.GetLastVideoLayersAllocation();
4486
4487 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4488 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4489 .target_bitrate_per_temporal_layer,
4490 SizeIs(2));
4491 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4492 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4493 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4494 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4495 .target_bitrate_per_temporal_layer,
4496 SizeIs(2));
4497 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4498 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4499 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4500
4501 // Since full SVC is used, expect the top layer to utilize the full target
4502 // rate.
4503 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4504 .target_bitrate_per_temporal_layer[1],
4505 DataRate::BitsPerSec(kTargetBitrateBps));
4506 video_stream_encoder_->Stop();
4507}
4508
4509TEST_F(VideoStreamEncoderTest,
4510 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4511 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4512 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004513 VideoEncoderConfig video_encoder_config;
4514 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4515 /* num_streams*/ 1, &video_encoder_config);
4516 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4517 video_encoder_config.content_type =
4518 VideoEncoderConfig::ContentType::kRealtimeVideo;
4519 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4520 vp9_settings.numberOfSpatialLayers = 2;
4521 vp9_settings.numberOfTemporalLayers = 2;
4522 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4523 vp9_settings.automaticResizeOn = false;
4524 video_encoder_config.encoder_specific_settings =
4525 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4526 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004527 ConfigureEncoder(std::move(video_encoder_config),
4528 VideoStreamEncoder::BitrateAllocationCallbackType::
4529 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004530
4531 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4532 DataRate::BitsPerSec(kTargetBitrateBps),
4533 DataRate::BitsPerSec(kTargetBitrateBps),
4534 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4535
4536 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4537 WaitForEncodedFrame(CurrentTimeMs());
4538 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4539 VideoLayersAllocation last_layer_allocation =
4540 sink_.GetLastVideoLayersAllocation();
4541
4542 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4543 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4544 .target_bitrate_per_temporal_layer,
4545 SizeIs(1));
4546 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4547 .target_bitrate_per_temporal_layer,
4548 SizeIs(1));
4549 // Since full SVC is used, expect the top layer to utilize the full target
4550 // rate.
4551 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4552 .target_bitrate_per_temporal_layer[0],
4553 DataRate::BitsPerSec(kTargetBitrateBps));
4554 video_stream_encoder_->Stop();
4555}
4556
4557TEST_F(VideoStreamEncoderTest,
4558 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4559 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4560 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004561 VideoEncoderConfig video_encoder_config;
4562 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4563 /* num_streams*/ 1, &video_encoder_config);
4564 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4565 video_encoder_config.content_type =
4566 VideoEncoderConfig::ContentType::kRealtimeVideo;
4567 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4568 vp9_settings.numberOfSpatialLayers = 2;
4569 vp9_settings.numberOfTemporalLayers = 2;
4570 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4571 vp9_settings.automaticResizeOn = false;
4572 video_encoder_config.encoder_specific_settings =
4573 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4574 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004575 ConfigureEncoder(std::move(video_encoder_config),
4576 VideoStreamEncoder::BitrateAllocationCallbackType::
4577 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004578
4579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4580 DataRate::BitsPerSec(kTargetBitrateBps),
4581 DataRate::BitsPerSec(kTargetBitrateBps),
4582 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4583
4584 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4585 WaitForEncodedFrame(CurrentTimeMs());
4586 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4587 VideoLayersAllocation last_layer_allocation =
4588 sink_.GetLastVideoLayersAllocation();
4589
4590 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4591 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4592 .target_bitrate_per_temporal_layer,
4593 SizeIs(2));
4594 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4595 .target_bitrate_per_temporal_layer,
4596 SizeIs(2));
4597 // Since KSVC is, spatial layers are independend except on key frames.
4598 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4599 .target_bitrate_per_temporal_layer[1],
4600 DataRate::BitsPerSec(kTargetBitrateBps));
4601 video_stream_encoder_->Stop();
4602}
4603
4604TEST_F(VideoStreamEncoderTest,
4605 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4606 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4607 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4608 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004609 VideoEncoderConfig video_encoder_config;
4610 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4611 /* num_streams*/ 1, &video_encoder_config);
4612 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4613 video_encoder_config.content_type =
4614 VideoEncoderConfig::ContentType::kRealtimeVideo;
4615 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4616 vp9_settings.numberOfSpatialLayers = 3;
4617 vp9_settings.numberOfTemporalLayers = 2;
4618 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4619 vp9_settings.automaticResizeOn = false;
4620 video_encoder_config.encoder_specific_settings =
4621 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4622 vp9_settings);
4623 // Simulcast layers are used for enabling/disabling streams.
4624 video_encoder_config.simulcast_layers.resize(3);
4625 video_encoder_config.simulcast_layers[0].active = false;
4626 video_encoder_config.simulcast_layers[1].active = true;
4627 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004628 ConfigureEncoder(std::move(video_encoder_config),
4629 VideoStreamEncoder::BitrateAllocationCallbackType::
4630 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004631
4632 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4633 DataRate::BitsPerSec(kTargetBitrateBps),
4634 DataRate::BitsPerSec(kTargetBitrateBps),
4635 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4636
4637 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4638 WaitForEncodedFrame(CurrentTimeMs());
4639 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4640 VideoLayersAllocation last_layer_allocation =
4641 sink_.GetLastVideoLayersAllocation();
4642
4643 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4644 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4645 .target_bitrate_per_temporal_layer,
4646 SizeIs(2));
4647 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4648 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4649
4650 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4651 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4652 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4653 .target_bitrate_per_temporal_layer,
4654 SizeIs(2));
4655 // Since full SVC is used, expect the top layer to utilize the full target
4656 // rate.
4657 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4658 .target_bitrate_per_temporal_layer[1],
4659 DataRate::BitsPerSec(kTargetBitrateBps));
4660 video_stream_encoder_->Stop();
4661}
4662
4663TEST_F(VideoStreamEncoderTest,
4664 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4665 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4666 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4667 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004668 VideoEncoderConfig video_encoder_config;
4669 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4670 /* num_streams*/ 1, &video_encoder_config);
4671 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4672 video_encoder_config.content_type =
4673 VideoEncoderConfig::ContentType::kRealtimeVideo;
4674 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4675 vp9_settings.numberOfSpatialLayers = 3;
4676 vp9_settings.numberOfTemporalLayers = 2;
4677 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4678 vp9_settings.automaticResizeOn = false;
4679 video_encoder_config.encoder_specific_settings =
4680 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4681 vp9_settings);
4682 // Simulcast layers are used for enabling/disabling streams.
4683 video_encoder_config.simulcast_layers.resize(3);
4684 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004685 ConfigureEncoder(std::move(video_encoder_config),
4686 VideoStreamEncoder::BitrateAllocationCallbackType::
4687 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004688
4689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4690 DataRate::BitsPerSec(kTargetBitrateBps),
4691 DataRate::BitsPerSec(kTargetBitrateBps),
4692 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4693
4694 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4695 WaitForEncodedFrame(CurrentTimeMs());
4696 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4697 VideoLayersAllocation last_layer_allocation =
4698 sink_.GetLastVideoLayersAllocation();
4699
4700 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4701 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4702 .target_bitrate_per_temporal_layer,
4703 SizeIs(2));
4704 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4705 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4706
4707 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4708 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4709 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4710 .target_bitrate_per_temporal_layer,
4711 SizeIs(2));
4712 video_stream_encoder_->Stop();
4713}
4714
4715TEST_F(VideoStreamEncoderTest,
4716 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4717 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004720 VideoEncoderConfig video_encoder_config;
4721 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4722 /* num_streams*/ 1, &video_encoder_config);
4723 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4724 video_encoder_config.content_type =
4725 VideoEncoderConfig::ContentType::kRealtimeVideo;
4726 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4727 vp9_settings.numberOfSpatialLayers = 3;
4728 vp9_settings.numberOfTemporalLayers = 2;
4729 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4730 vp9_settings.automaticResizeOn = false;
4731 video_encoder_config.encoder_specific_settings =
4732 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4733 vp9_settings);
4734 // Simulcast layers are used for enabling/disabling streams.
4735 video_encoder_config.simulcast_layers.resize(3);
4736 video_encoder_config.simulcast_layers[0].active = false;
4737 video_encoder_config.simulcast_layers[1].active = false;
4738 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004739 ConfigureEncoder(std::move(video_encoder_config),
4740 VideoStreamEncoder::BitrateAllocationCallbackType::
4741 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004742
4743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4744 DataRate::BitsPerSec(kTargetBitrateBps),
4745 DataRate::BitsPerSec(kTargetBitrateBps),
4746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4747
4748 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4749 WaitForEncodedFrame(CurrentTimeMs());
4750 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4751 VideoLayersAllocation last_layer_allocation =
4752 sink_.GetLastVideoLayersAllocation();
4753
4754 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4755 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4756 .target_bitrate_per_temporal_layer,
4757 SizeIs(2));
4758 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4759 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4760 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4761 .target_bitrate_per_temporal_layer[1],
4762 DataRate::BitsPerSec(kTargetBitrateBps));
4763 video_stream_encoder_->Stop();
4764}
4765
4766TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4767 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004768 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004769 kVideoLayersAllocation);
4770 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4771 DataRate::BitsPerSec(kTargetBitrateBps),
4772 DataRate::BitsPerSec(kTargetBitrateBps),
4773 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4774
4775 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4776 WaitForEncodedFrame(CurrentTimeMs());
4777 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4778 VideoLayersAllocation last_layer_allocation =
4779 sink_.GetLastVideoLayersAllocation();
4780
4781 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4782 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4783 .target_bitrate_per_temporal_layer,
4784 SizeIs(1));
4785 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4786 .target_bitrate_per_temporal_layer[0],
4787 DataRate::BitsPerSec(kTargetBitrateBps));
4788 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4789 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4790 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4791 video_stream_encoder_->Stop();
4792}
4793
4794TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004795 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4796 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004797 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004798 kVideoLayersAllocation);
4799
4800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4801 DataRate::BitsPerSec(kLowTargetBitrateBps),
4802 DataRate::BitsPerSec(kLowTargetBitrateBps),
4803 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4804
4805 video_source_.IncomingCapturedFrame(
4806 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4807 WaitForEncodedFrame(CurrentTimeMs());
4808 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4809 VideoLayersAllocation last_layer_allocation =
4810 sink_.GetLastVideoLayersAllocation();
4811 // kLowTargetBitrateBps is only enough for one spatial layer.
4812 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4813 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4814 .target_bitrate_per_temporal_layer[0],
4815 DataRate::BitsPerSec(kLowTargetBitrateBps));
4816
4817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4818 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4819 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4820 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4821 video_source_.IncomingCapturedFrame(
4822 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4823 WaitForEncodedFrame(CurrentTimeMs());
4824
4825 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4826 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4827 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4828 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4829 .target_bitrate_per_temporal_layer[0],
4830 DataRate::Zero());
4831
4832 video_stream_encoder_->Stop();
4833}
4834
Per Kjellander4190ce92020-12-15 17:24:55 +01004835TEST_F(VideoStreamEncoderTest,
4836 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4837 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004838 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004839 kVideoLayersAllocation);
4840
4841 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4842 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4843 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4844 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4845
4846 video_source_.IncomingCapturedFrame(
4847 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4848 WaitForEncodedFrame(CurrentTimeMs());
4849 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4850 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4851 SizeIs(2));
4852 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4853 codec_width_);
4854 EXPECT_EQ(
4855 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4856 codec_height_);
4857
4858 video_source_.IncomingCapturedFrame(
4859 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4860 WaitForEncodedFrame(CurrentTimeMs());
4861 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4862 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4863 SizeIs(2));
4864 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4865 codec_width_ / 2);
4866 EXPECT_EQ(
4867 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4868 codec_height_ / 2);
4869
4870 video_stream_encoder_->Stop();
4871}
4872
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004873TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4874 // 2 TLs configured, temporal layers supported by encoder.
4875 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004876 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004877 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004878 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004879 fake_encoder_.SetTemporalLayersSupported(0, true);
4880
4881 // Bitrate allocated across temporal layers.
4882 const int kTl0Bps = kTargetBitrateBps *
4883 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004884 kNumTemporalLayers, /*temporal_id*/ 0,
4885 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004886 const int kTl1Bps = kTargetBitrateBps *
4887 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004888 kNumTemporalLayers, /*temporal_id*/ 1,
4889 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004890 VideoBitrateAllocation expected_bitrate;
4891 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4892 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4893
4894 VerifyAllocatedBitrate(expected_bitrate);
4895 video_stream_encoder_->Stop();
4896}
4897
4898TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4899 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004900 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004901 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004902 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004903 fake_encoder_.SetTemporalLayersSupported(0, false);
4904
4905 // Temporal layers not supported by the encoder.
4906 // Total bitrate should be at ti:0.
4907 VideoBitrateAllocation expected_bitrate;
4908 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4909
4910 VerifyAllocatedBitrate(expected_bitrate);
4911 video_stream_encoder_->Stop();
4912}
4913
4914TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004915 webrtc::test::ScopedFieldTrials field_trials(
4916 "WebRTC-Video-QualityScalerSettings/"
4917 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4918 // Reset encoder for field trials to take effect.
4919 ConfigureEncoder(video_encoder_config_.Copy());
4920
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004921 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004922 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004923 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004924 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004925 fake_encoder_.SetTemporalLayersSupported(0, true);
4926 fake_encoder_.SetTemporalLayersSupported(1, false);
4927
4928 const int kS0Bps = 150000;
4929 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004930 kS0Bps *
4931 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4932 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004933 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004934 kS0Bps *
4935 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4936 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004937 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4938 // Temporal layers not supported by si:1.
4939 VideoBitrateAllocation expected_bitrate;
4940 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4941 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4942 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4943
4944 VerifyAllocatedBitrate(expected_bitrate);
4945 video_stream_encoder_->Stop();
4946}
4947
Niels Möller7dc26b72017-12-06 10:27:48 +01004948TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4949 const int kFrameWidth = 1280;
4950 const int kFrameHeight = 720;
4951 const int kFramerate = 24;
4952
Henrik Boström381d1092020-05-12 18:49:07 +02004953 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004954 DataRate::BitsPerSec(kTargetBitrateBps),
4955 DataRate::BitsPerSec(kTargetBitrateBps),
4956 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004957 test::FrameForwarder source;
4958 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004959 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004960
4961 // Insert a single frame, triggering initial configuration.
4962 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4963 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4964
4965 EXPECT_EQ(
4966 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4967 kDefaultFramerate);
4968
4969 // Trigger reconfigure encoder (without resetting the entire instance).
4970 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004971 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4972 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004973 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004974 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004975 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4977
4978 // Detector should be updated with fps limit from codec config.
4979 EXPECT_EQ(
4980 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4981 kFramerate);
4982
4983 // Trigger overuse, max framerate should be reduced.
4984 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4985 stats.input_frame_rate = kFramerate;
4986 stats_proxy_->SetMockStats(stats);
4987 video_stream_encoder_->TriggerCpuOveruse();
4988 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4989 int adapted_framerate =
4990 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4991 EXPECT_LT(adapted_framerate, kFramerate);
4992
4993 // Trigger underuse, max framerate should go back to codec configured fps.
4994 // Set extra low fps, to make sure it's actually reset, not just incremented.
4995 stats = stats_proxy_->GetStats();
4996 stats.input_frame_rate = adapted_framerate / 2;
4997 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004998 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004999 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5000 EXPECT_EQ(
5001 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5002 kFramerate);
5003
5004 video_stream_encoder_->Stop();
5005}
5006
5007TEST_F(VideoStreamEncoderTest,
5008 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5009 const int kFrameWidth = 1280;
5010 const int kFrameHeight = 720;
5011 const int kLowFramerate = 15;
5012 const int kHighFramerate = 25;
5013
Henrik Boström381d1092020-05-12 18:49:07 +02005014 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005015 DataRate::BitsPerSec(kTargetBitrateBps),
5016 DataRate::BitsPerSec(kTargetBitrateBps),
5017 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005018 test::FrameForwarder source;
5019 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005020 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005021
5022 // Trigger initial configuration.
5023 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005024 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5025 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005026 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005027 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005028 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005029 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005030 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5031
5032 EXPECT_EQ(
5033 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5034 kLowFramerate);
5035
5036 // Trigger overuse, max framerate should be reduced.
5037 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5038 stats.input_frame_rate = kLowFramerate;
5039 stats_proxy_->SetMockStats(stats);
5040 video_stream_encoder_->TriggerCpuOveruse();
5041 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5042 int adapted_framerate =
5043 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5044 EXPECT_LT(adapted_framerate, kLowFramerate);
5045
5046 // Reconfigure the encoder with a new (higher max framerate), max fps should
5047 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005048 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005049 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5050 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005051 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005052 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5053
5054 EXPECT_EQ(
5055 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5056 adapted_framerate);
5057
5058 // Trigger underuse, max framerate should go back to codec configured fps.
5059 stats = stats_proxy_->GetStats();
5060 stats.input_frame_rate = adapted_framerate;
5061 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005062 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005063 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5064 EXPECT_EQ(
5065 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5066 kHighFramerate);
5067
5068 video_stream_encoder_->Stop();
5069}
5070
mflodmancc3d4422017-08-03 08:27:51 -07005071TEST_F(VideoStreamEncoderTest,
5072 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005073 const int kFrameWidth = 1280;
5074 const int kFrameHeight = 720;
5075 const int kFramerate = 24;
5076
Henrik Boström381d1092020-05-12 18:49:07 +02005077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005078 DataRate::BitsPerSec(kTargetBitrateBps),
5079 DataRate::BitsPerSec(kTargetBitrateBps),
5080 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005081 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005082 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005083 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005084
5085 // Trigger initial configuration.
5086 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005087 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5088 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005089 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005090 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005091 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005092 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005093 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005094
Niels Möller7dc26b72017-12-06 10:27:48 +01005095 EXPECT_EQ(
5096 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5097 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005098
5099 // Trigger overuse, max framerate should be reduced.
5100 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5101 stats.input_frame_rate = kFramerate;
5102 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005103 video_stream_encoder_->TriggerCpuOveruse();
5104 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005105 int adapted_framerate =
5106 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005107 EXPECT_LT(adapted_framerate, kFramerate);
5108
5109 // Change degradation preference to not enable framerate scaling. Target
5110 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005111 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005112 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005113 EXPECT_EQ(
5114 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5115 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005116
mflodmancc3d4422017-08-03 08:27:51 -07005117 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005118}
5119
mflodmancc3d4422017-08-03 08:27:51 -07005120TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005121 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005122 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005123 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5124 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5125 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005126 const int kWidth = 640;
5127 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005128
asaperssonfab67072017-04-04 05:51:49 -07005129 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005130
5131 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005132 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005133
5134 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005135 EXPECT_TRUE_WAIT(
5136 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005137
sprangc5d62e22017-04-02 23:53:04 -07005138 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005139
asaperssonfab67072017-04-04 05:51:49 -07005140 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005141 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005142 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005143
5144 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005145 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005146
Henrik Boström2671dac2020-05-19 16:29:09 +02005147 EXPECT_TRUE_WAIT(
5148 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005149
mflodmancc3d4422017-08-03 08:27:51 -07005150 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005151}
5152
mflodmancc3d4422017-08-03 08:27:51 -07005153TEST_F(VideoStreamEncoderTest,
5154 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005155 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005156 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005157 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5158 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5159 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005160 const int kWidth = 640;
5161 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005162
5163 // We expect the n initial frames to get dropped.
5164 int i;
5165 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005166 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005167 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005168 }
5169 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005170 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005171 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005172
5173 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005174 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005175
mflodmancc3d4422017-08-03 08:27:51 -07005176 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005177}
5178
mflodmancc3d4422017-08-03 08:27:51 -07005179TEST_F(VideoStreamEncoderTest,
5180 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005181 const int kWidth = 640;
5182 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005184 DataRate::BitsPerSec(kLowTargetBitrateBps),
5185 DataRate::BitsPerSec(kLowTargetBitrateBps),
5186 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005187
5188 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005189 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005190 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005191
asaperssonfab67072017-04-04 05:51:49 -07005192 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005193 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005194 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005195
mflodmancc3d4422017-08-03 08:27:51 -07005196 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005197}
5198
mflodmancc3d4422017-08-03 08:27:51 -07005199TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005200 const int kWidth = 640;
5201 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005202 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005203
5204 VideoEncoderConfig video_encoder_config;
5205 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5206 // Make format different, to force recreation of encoder.
5207 video_encoder_config.video_format.parameters["foo"] = "foo";
5208 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005209 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005210 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005211 DataRate::BitsPerSec(kLowTargetBitrateBps),
5212 DataRate::BitsPerSec(kLowTargetBitrateBps),
5213 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005214
kthelgasonb83797b2017-02-14 11:57:25 -08005215 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005216 video_stream_encoder_->SetSource(&video_source_,
5217 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005218
asaperssonfab67072017-04-04 05:51:49 -07005219 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005220 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005221 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005222
mflodmancc3d4422017-08-03 08:27:51 -07005223 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005224 fake_encoder_.SetQualityScaling(true);
5225}
5226
Åsa Persson139f4dc2019-08-02 09:29:58 +02005227TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5228 webrtc::test::ScopedFieldTrials field_trials(
5229 "WebRTC-Video-QualityScalerSettings/"
5230 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5231 // Reset encoder for field trials to take effect.
5232 ConfigureEncoder(video_encoder_config_.Copy());
5233 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5234 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5235 const int kWidth = 640;
5236 const int kHeight = 360;
5237
Henrik Boström381d1092020-05-12 18:49:07 +02005238 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005239 DataRate::BitsPerSec(kTargetBitrateBps),
5240 DataRate::BitsPerSec(kTargetBitrateBps),
5241 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005242 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5243 // Frame should not be dropped.
5244 WaitForEncodedFrame(1);
5245
Henrik Boström381d1092020-05-12 18:49:07 +02005246 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005247 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5248 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5249 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005250 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5251 // Frame should not be dropped.
5252 WaitForEncodedFrame(2);
5253
Henrik Boström381d1092020-05-12 18:49:07 +02005254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005255 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5256 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5257 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005258 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5259 // Expect to drop this frame, the wait should time out.
5260 ExpectDroppedFrame();
5261
5262 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005263 EXPECT_TRUE_WAIT(
5264 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005265 video_stream_encoder_->Stop();
5266}
5267
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005268TEST_F(VideoStreamEncoderTest,
5269 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5270 webrtc::test::ScopedFieldTrials field_trials(
5271 "WebRTC-Video-QualityScalerSettings/"
5272 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5273 fake_encoder_.SetQualityScaling(false);
5274 ConfigureEncoder(video_encoder_config_.Copy());
5275 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5276 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5277 const int kWidth = 640;
5278 const int kHeight = 360;
5279
5280 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5281 DataRate::BitsPerSec(kTargetBitrateBps),
5282 DataRate::BitsPerSec(kTargetBitrateBps),
5283 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5284 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5285 // Frame should not be dropped.
5286 WaitForEncodedFrame(1);
5287
5288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5289 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5290 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5291 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5292 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5293 // Frame should not be dropped.
5294 WaitForEncodedFrame(2);
5295
5296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5297 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5298 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5299 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5300 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5301 // Not dropped since quality scaling is disabled.
5302 WaitForEncodedFrame(3);
5303
5304 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005305 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005306 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5307
5308 video_stream_encoder_->Stop();
5309}
5310
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005311TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5312 const int kLowTargetBitrateBps = 400000;
5313 // Set simulcast.
5314 ResetEncoder("VP8", 3, 1, 1, false);
5315 fake_encoder_.SetQualityScaling(true);
5316 const int kWidth = 1280;
5317 const int kHeight = 720;
5318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5319 DataRate::BitsPerSec(kLowTargetBitrateBps),
5320 DataRate::BitsPerSec(kLowTargetBitrateBps),
5321 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5322 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5323 // Frame should not be dropped.
5324 WaitForEncodedFrame(1);
5325
5326 // Trigger QVGA "singlecast"
5327 // Update the config.
5328 VideoEncoderConfig video_encoder_config;
5329 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5330 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005331 video_encoder_config.video_stream_factory =
5332 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5333 "VP8", /*max qp*/ 56, /*screencast*/ false,
5334 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005335 for (auto& layer : video_encoder_config.simulcast_layers) {
5336 layer.num_temporal_layers = 1;
5337 layer.max_framerate = kDefaultFramerate;
5338 }
5339 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5340 video_encoder_config.content_type =
5341 VideoEncoderConfig::ContentType::kRealtimeVideo;
5342
5343 video_encoder_config.simulcast_layers[0].active = true;
5344 video_encoder_config.simulcast_layers[1].active = false;
5345 video_encoder_config.simulcast_layers[2].active = false;
5346
5347 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5348 kMaxPayloadLength);
5349 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5350
5351 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5352 // Frame should not be dropped.
5353 WaitForEncodedFrame(2);
5354
5355 // Trigger HD "singlecast"
5356 video_encoder_config.simulcast_layers[0].active = false;
5357 video_encoder_config.simulcast_layers[1].active = false;
5358 video_encoder_config.simulcast_layers[2].active = true;
5359
5360 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5361 kMaxPayloadLength);
5362 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5363
5364 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5365 // Frame should be dropped because of initial frame drop.
5366 ExpectDroppedFrame();
5367
5368 // Expect the sink_wants to specify a scaled frame.
5369 EXPECT_TRUE_WAIT(
5370 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5371 video_stream_encoder_->Stop();
5372}
5373
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005374TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5375 const int kLowTargetBitrateBps = 400000;
5376 // Set simulcast.
5377 ResetEncoder("VP9", 1, 1, 3, false);
5378 fake_encoder_.SetQualityScaling(true);
5379 const int kWidth = 1280;
5380 const int kHeight = 720;
5381 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5382 DataRate::BitsPerSec(kLowTargetBitrateBps),
5383 DataRate::BitsPerSec(kLowTargetBitrateBps),
5384 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5385 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5386 // Frame should not be dropped.
5387 WaitForEncodedFrame(1);
5388
5389 // Trigger QVGA "singlecast"
5390 // Update the config.
5391 VideoEncoderConfig video_encoder_config;
5392 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5393 &video_encoder_config);
5394 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5395 vp9_settings.numberOfSpatialLayers = 3;
5396 // Since only one layer is active - automatic resize should be enabled.
5397 vp9_settings.automaticResizeOn = true;
5398 video_encoder_config.encoder_specific_settings =
5399 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5400 vp9_settings);
5401 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5402 video_encoder_config.content_type =
5403 VideoEncoderConfig::ContentType::kRealtimeVideo;
5404 // Currently simulcast layers |active| flags are used to inidicate
5405 // which SVC layers are active.
5406 video_encoder_config.simulcast_layers.resize(3);
5407
5408 video_encoder_config.simulcast_layers[0].active = true;
5409 video_encoder_config.simulcast_layers[1].active = false;
5410 video_encoder_config.simulcast_layers[2].active = false;
5411
5412 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5413 kMaxPayloadLength);
5414 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5415
5416 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5417 // Frame should not be dropped.
5418 WaitForEncodedFrame(2);
5419
5420 // Trigger HD "singlecast"
5421 video_encoder_config.simulcast_layers[0].active = false;
5422 video_encoder_config.simulcast_layers[1].active = false;
5423 video_encoder_config.simulcast_layers[2].active = true;
5424
5425 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5426 kMaxPayloadLength);
5427 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5428
5429 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5430 // Frame should be dropped because of initial frame drop.
5431 ExpectDroppedFrame();
5432
5433 // Expect the sink_wants to specify a scaled frame.
5434 EXPECT_TRUE_WAIT(
5435 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5436 video_stream_encoder_->Stop();
5437}
5438
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005439TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005440 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5441 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5442 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5443 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5444 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5445 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5446 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5447 fake_encoder_.SetResolutionBitrateLimits(
5448 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5449
5450 VideoEncoderConfig video_encoder_config;
5451 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5452 &video_encoder_config);
5453 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5454 vp9_settings.numberOfSpatialLayers = 3;
5455 // Since only one layer is active - automatic resize should be enabled.
5456 vp9_settings.automaticResizeOn = true;
5457 video_encoder_config.encoder_specific_settings =
5458 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5459 vp9_settings);
5460 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5461 video_encoder_config.content_type =
5462 VideoEncoderConfig::ContentType::kRealtimeVideo;
5463 // Simulcast layers are used to indicate which spatial layers are active.
5464 video_encoder_config.simulcast_layers.resize(3);
5465 video_encoder_config.simulcast_layers[0].active = false;
5466 video_encoder_config.simulcast_layers[1].active = true;
5467 video_encoder_config.simulcast_layers[2].active = false;
5468
5469 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5470 kMaxPayloadLength);
5471 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5472
5473 // The encoder bitrate limits for 360p should be used.
5474 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5475 EXPECT_FALSE(WaitForFrame(1000));
5476 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5477 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5478 VideoCodecType::kVideoCodecVP9);
5479 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5480 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5481 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5482 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5483 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5484 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5485 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5486 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5487
5488 // The encoder bitrate limits for 270p should be used.
5489 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5490 EXPECT_FALSE(WaitForFrame(1000));
5491 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5492 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5493 VideoCodecType::kVideoCodecVP9);
5494 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5495 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5496 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5497 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5498 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5499 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5500 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5501 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5502
5503 video_stream_encoder_->Stop();
5504}
5505
5506TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005507 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5508 VideoEncoderConfig video_encoder_config;
5509 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5510 &video_encoder_config);
5511 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5512 vp9_settings.numberOfSpatialLayers = 3;
5513 // Since only one layer is active - automatic resize should be enabled.
5514 vp9_settings.automaticResizeOn = true;
5515 video_encoder_config.encoder_specific_settings =
5516 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5517 vp9_settings);
5518 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5519 video_encoder_config.content_type =
5520 VideoEncoderConfig::ContentType::kRealtimeVideo;
5521 // Simulcast layers are used to indicate which spatial layers are active.
5522 video_encoder_config.simulcast_layers.resize(3);
5523 video_encoder_config.simulcast_layers[0].active = false;
5524 video_encoder_config.simulcast_layers[1].active = true;
5525 video_encoder_config.simulcast_layers[2].active = false;
5526
5527 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5528 kMaxPayloadLength);
5529 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5530
5531 // The default bitrate limits for 360p should be used.
5532 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005533 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5534 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005535 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5536 EXPECT_FALSE(WaitForFrame(1000));
5537 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5538 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5539 VideoCodecType::kVideoCodecVP9);
5540 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5541 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5542 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5543 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5544 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5545 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5546 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5547 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5548
5549 // The default bitrate limits for 270p should be used.
5550 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005551 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5552 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005553 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5554 EXPECT_FALSE(WaitForFrame(1000));
5555 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5556 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5557 VideoCodecType::kVideoCodecVP9);
5558 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5559 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5560 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5561 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5562 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5563 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5564 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5565 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5566
5567 video_stream_encoder_->Stop();
5568}
5569
5570TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5571 webrtc::test::ScopedFieldTrials field_trials(
5572 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5573 VideoEncoderConfig video_encoder_config;
5574 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5575 &video_encoder_config);
5576 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5577 vp9_settings.numberOfSpatialLayers = 3;
5578 // Since only one layer is active - automatic resize should be enabled.
5579 vp9_settings.automaticResizeOn = true;
5580 video_encoder_config.encoder_specific_settings =
5581 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5582 vp9_settings);
5583 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5584 video_encoder_config.content_type =
5585 VideoEncoderConfig::ContentType::kRealtimeVideo;
5586 // Simulcast layers are used to indicate which spatial layers are active.
5587 video_encoder_config.simulcast_layers.resize(3);
5588 video_encoder_config.simulcast_layers[0].active = false;
5589 video_encoder_config.simulcast_layers[1].active = true;
5590 video_encoder_config.simulcast_layers[2].active = false;
5591
5592 // Reset encoder for field trials to take effect.
5593 ConfigureEncoder(video_encoder_config.Copy());
5594
5595 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5596 kMaxPayloadLength);
5597 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5598
5599 // The default bitrate limits for 360p should not be used.
5600 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005601 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5602 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005603 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5604 EXPECT_FALSE(WaitForFrame(1000));
5605 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005606 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005607 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5608 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5609 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5610 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5611 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5612 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5613
5614 video_stream_encoder_->Stop();
5615}
5616
5617TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5618 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5619 /*num_spatial_layers=*/1, /*screenshare=*/false);
5620
5621 // The default singlecast bitrate limits for 720p should not be used.
5622 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005623 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5624 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005625 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5626 EXPECT_FALSE(WaitForFrame(1000));
5627 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5628 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5629 VideoCodecType::kVideoCodecVP9);
5630 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5631 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5632 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5633 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5634 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5635 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5636
5637 video_stream_encoder_->Stop();
5638}
5639
5640TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005641 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5642 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5643 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5644 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5645 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5646 fake_encoder_.SetResolutionBitrateLimits(
5647 {kEncoderLimits180p, kEncoderLimits720p});
5648
5649 VideoEncoderConfig video_encoder_config;
5650 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5651 &video_encoder_config);
5652 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5653 vp9_settings.numberOfSpatialLayers = 3;
5654 // Since only one layer is active - automatic resize should be enabled.
5655 vp9_settings.automaticResizeOn = true;
5656 video_encoder_config.encoder_specific_settings =
5657 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5658 vp9_settings);
5659 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5660 video_encoder_config.content_type =
5661 VideoEncoderConfig::ContentType::kRealtimeVideo;
5662 // Simulcast layers are used to indicate which spatial layers are active.
5663 video_encoder_config.simulcast_layers.resize(3);
5664 video_encoder_config.simulcast_layers[0].active = true;
5665 video_encoder_config.simulcast_layers[1].active = false;
5666 video_encoder_config.simulcast_layers[2].active = false;
5667
5668 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5669 kMaxPayloadLength);
5670 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5671
5672 // Limits not applied on lowest stream, limits for 180p should not be used.
5673 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5674 EXPECT_FALSE(WaitForFrame(1000));
5675 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5676 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5677 VideoCodecType::kVideoCodecVP9);
5678 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5679 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5680 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5681 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5682 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5683 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5684 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5685 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5686
5687 video_stream_encoder_->Stop();
5688}
5689
5690TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005691 InitialFrameDropActivatesWhenResolutionIncreases) {
5692 const int kWidth = 640;
5693 const int kHeight = 360;
5694
5695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5696 DataRate::BitsPerSec(kTargetBitrateBps),
5697 DataRate::BitsPerSec(kTargetBitrateBps),
5698 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5699 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5700 // Frame should not be dropped.
5701 WaitForEncodedFrame(1);
5702
5703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5704 DataRate::BitsPerSec(kLowTargetBitrateBps),
5705 DataRate::BitsPerSec(kLowTargetBitrateBps),
5706 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5707 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5708 // Frame should not be dropped, bitrate not too low for frame.
5709 WaitForEncodedFrame(2);
5710
5711 // Incoming resolution increases.
5712 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5713 // Expect to drop this frame, bitrate too low for frame.
5714 ExpectDroppedFrame();
5715
5716 // Expect the sink_wants to specify a scaled frame.
5717 EXPECT_TRUE_WAIT(
5718 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5719 video_stream_encoder_->Stop();
5720}
5721
5722TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5723 const int kWidth = 640;
5724 const int kHeight = 360;
5725 // So that quality scaling doesn't happen by itself.
5726 fake_encoder_.SetQp(kQpHigh);
5727
5728 AdaptingFrameForwarder source(&time_controller_);
5729 source.set_adaptation_enabled(true);
5730 video_stream_encoder_->SetSource(
5731 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5732
5733 int timestamp = 1;
5734
5735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5736 DataRate::BitsPerSec(kTargetBitrateBps),
5737 DataRate::BitsPerSec(kTargetBitrateBps),
5738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5739 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5740 WaitForEncodedFrame(timestamp);
5741 timestamp += 9000;
5742 // Long pause to disable all first BWE drop logic.
5743 AdvanceTime(TimeDelta::Millis(1000));
5744
5745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5746 DataRate::BitsPerSec(kLowTargetBitrateBps),
5747 DataRate::BitsPerSec(kLowTargetBitrateBps),
5748 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5749 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5750 // Not dropped frame, as initial frame drop is disabled by now.
5751 WaitForEncodedFrame(timestamp);
5752 timestamp += 9000;
5753 AdvanceTime(TimeDelta::Millis(100));
5754
5755 // Quality adaptation down.
5756 video_stream_encoder_->TriggerQualityLow();
5757
5758 // Adaptation has an effect.
5759 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5760 5000);
5761
5762 // Frame isn't dropped as initial frame dropper is disabled.
5763 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5764 WaitForEncodedFrame(timestamp);
5765 timestamp += 9000;
5766 AdvanceTime(TimeDelta::Millis(100));
5767
5768 // Quality adaptation up.
5769 video_stream_encoder_->TriggerQualityHigh();
5770
5771 // Adaptation has an effect.
5772 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5773 5000);
5774
5775 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5776 // Frame should not be dropped, as initial framedropper is off.
5777 WaitForEncodedFrame(timestamp);
5778
5779 video_stream_encoder_->Stop();
5780}
5781
Åsa Persson7f354f82021-02-04 15:52:15 +01005782TEST_F(VideoStreamEncoderTest,
5783 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5784 const int kMinStartBps360p = 222000;
5785 fake_encoder_.SetResolutionBitrateLimits(
5786 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5787 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5788 800000)});
5789
5790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5791 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5792 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5793 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5794 0, 0, 0);
5795 // Frame should not be dropped, bitrate not too low for frame.
5796 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5797 WaitForEncodedFrame(1);
5798
5799 // Incoming resolution increases, initial frame drop activates.
5800 // Frame should be dropped, link allocation too low for frame.
5801 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5802 ExpectDroppedFrame();
5803
5804 // Expect sink_wants to specify a scaled frame.
5805 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5806 5000);
5807 video_stream_encoder_->Stop();
5808}
5809
5810TEST_F(VideoStreamEncoderTest,
5811 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5812 const int kMinStartBps360p = 222000;
5813 fake_encoder_.SetResolutionBitrateLimits(
5814 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5815 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5816 800000)});
5817
5818 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5819 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5820 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5821 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5822 0, 0, 0);
5823 // Frame should not be dropped, bitrate not too low for frame.
5824 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5825 WaitForEncodedFrame(1);
5826
5827 // Incoming resolution increases, initial frame drop activates.
5828 // Frame should be dropped, link allocation not too low for frame.
5829 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5830 WaitForEncodedFrame(2);
5831
5832 video_stream_encoder_->Stop();
5833}
5834
Åsa Perssone644a032019-11-08 15:56:00 +01005835TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5836 webrtc::test::ScopedFieldTrials field_trials(
5837 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5838
5839 // Reset encoder for field trials to take effect.
5840 VideoEncoderConfig config = video_encoder_config_.Copy();
5841 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005842 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005843 ConfigureEncoder(std::move(config));
5844 fake_encoder_.SetQp(kQpLow);
5845
5846 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005847 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005848 source.set_adaptation_enabled(true);
5849 video_stream_encoder_->SetSource(&source,
5850 DegradationPreference::MAINTAIN_FRAMERATE);
5851
5852 // Start at low bitrate.
5853 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5855 DataRate::BitsPerSec(kLowBitrateBps),
5856 DataRate::BitsPerSec(kLowBitrateBps),
5857 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005858
5859 // Expect first frame to be dropped and resolution to be limited.
5860 const int kWidth = 1280;
5861 const int kHeight = 720;
5862 const int64_t kFrameIntervalMs = 100;
5863 int64_t timestamp_ms = kFrameIntervalMs;
5864 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5865 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005866 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5867 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005868
5869 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5871 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005872
5873 // Insert frames and advance |min_duration_ms|.
5874 for (size_t i = 1; i <= 10; i++) {
5875 timestamp_ms += kFrameIntervalMs;
5876 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5877 WaitForEncodedFrame(timestamp_ms);
5878 }
5879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5880 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5881
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005882 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005883
5884 // Insert frame should trigger high BW and release quality limitation.
5885 timestamp_ms += kFrameIntervalMs;
5886 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5887 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005888 // The ramp-up code involves the adaptation queue, give it time to execute.
5889 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005890 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005891 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005892
5893 // Frame should not be adapted.
5894 timestamp_ms += kFrameIntervalMs;
5895 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5896 WaitForEncodedFrame(kWidth, kHeight);
5897 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5898
5899 video_stream_encoder_->Stop();
5900}
5901
mflodmancc3d4422017-08-03 08:27:51 -07005902TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005903 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005904 webrtc::test::ScopedFieldTrials field_trials(
5905 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005906 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005907 source.set_adaptation_enabled(true);
5908 video_stream_encoder_->SetSource(&source,
5909 DegradationPreference::MAINTAIN_FRAMERATE);
5910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5911 DataRate::BitsPerSec(kTargetBitrateBps),
5912 DataRate::BitsPerSec(kTargetBitrateBps),
5913 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5914 fake_encoder_.SetQp(kQpHigh + 1);
5915 const int kWidth = 1280;
5916 const int kHeight = 720;
5917 const int64_t kFrameIntervalMs = 100;
5918 int64_t timestamp_ms = kFrameIntervalMs;
5919 for (size_t i = 1; i <= 100; i++) {
5920 timestamp_ms += kFrameIntervalMs;
5921 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5922 WaitForEncodedFrame(timestamp_ms);
5923 }
5924 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5925 // for the first time.
5926 // TODO(eshr): We should avoid these waits by using threads with simulated
5927 // time.
5928 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5929 2000 * 2.5 * 2);
5930 timestamp_ms += kFrameIntervalMs;
5931 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5932 WaitForEncodedFrame(timestamp_ms);
5933 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5934 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5935 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5936
5937 // Disable Quality scaling by turning off scaler on the encoder and
5938 // reconfiguring.
5939 fake_encoder_.SetQualityScaling(false);
5940 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5941 kMaxPayloadLength);
5942 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005943 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005944 // Since we turned off the quality scaler, the adaptations made by it are
5945 // removed.
5946 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5947 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5948
5949 video_stream_encoder_->Stop();
5950}
5951
5952TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005953 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5954 const int kTooSmallWidth = 10;
5955 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005957 DataRate::BitsPerSec(kTargetBitrateBps),
5958 DataRate::BitsPerSec(kTargetBitrateBps),
5959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005960
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005961 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005962 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005963 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005964 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005965 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005966 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5967
5968 // Trigger adapt down, too small frame, expect no change.
5969 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005970 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005971 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005972 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005973 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5974 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5975
mflodmancc3d4422017-08-03 08:27:51 -07005976 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005977}
5978
mflodmancc3d4422017-08-03 08:27:51 -07005979TEST_F(VideoStreamEncoderTest,
5980 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005981 const int kTooSmallWidth = 10;
5982 const int kTooSmallHeight = 10;
5983 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005985 DataRate::BitsPerSec(kTargetBitrateBps),
5986 DataRate::BitsPerSec(kTargetBitrateBps),
5987 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005988
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005989 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07005990 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005991 video_stream_encoder_->SetSource(&source,
5992 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005993 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07005994 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5995 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5996
5997 // Trigger adapt down, expect limited framerate.
5998 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005999 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006000 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006001 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006002 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6003 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6004 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6005
6006 // Trigger adapt down, too small frame, expect no change.
6007 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006008 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006009 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006010 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006011 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6012 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6013 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6014
mflodmancc3d4422017-08-03 08:27:51 -07006015 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006016}
6017
mflodmancc3d4422017-08-03 08:27:51 -07006018TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006019 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006021 DataRate::BitsPerSec(kTargetBitrateBps),
6022 DataRate::BitsPerSec(kTargetBitrateBps),
6023 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006024 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006025 const int kFrameWidth = 1280;
6026 const int kFrameHeight = 720;
6027 video_source_.IncomingCapturedFrame(
6028 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006029 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006030 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006031}
6032
sprangb1ca0732017-02-01 08:38:12 -08006033// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006034TEST_F(VideoStreamEncoderTest,
6035 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006037 DataRate::BitsPerSec(kTargetBitrateBps),
6038 DataRate::BitsPerSec(kTargetBitrateBps),
6039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006040
6041 const int kFrameWidth = 1280;
6042 const int kFrameHeight = 720;
6043 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006044 // requested by
6045 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006046 video_source_.set_adaptation_enabled(true);
6047
6048 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006049 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006050 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006051
6052 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006053 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006054 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006055 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006056 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006057
asaperssonfab67072017-04-04 05:51:49 -07006058 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006059 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006060 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006061 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006062 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006063
mflodmancc3d4422017-08-03 08:27:51 -07006064 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006065}
sprangfe627f32017-03-29 08:24:59 -07006066
mflodmancc3d4422017-08-03 08:27:51 -07006067TEST_F(VideoStreamEncoderTest,
6068 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006069 const int kFrameWidth = 1280;
6070 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006071
Henrik Boström381d1092020-05-12 18:49:07 +02006072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006073 DataRate::BitsPerSec(kTargetBitrateBps),
6074 DataRate::BitsPerSec(kTargetBitrateBps),
6075 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006076 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006077 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006078 video_source_.set_adaptation_enabled(true);
6079
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006080 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006081
6082 video_source_.IncomingCapturedFrame(
6083 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006084 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006085
6086 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006087 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006088
6089 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006090 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006091 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006092 video_source_.IncomingCapturedFrame(
6093 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006094 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006095 }
6096
6097 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006098 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006099 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006100 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006101 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006102 video_source_.IncomingCapturedFrame(
6103 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006104 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006105 ++num_frames_dropped;
6106 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006107 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006108 }
6109 }
6110
sprang4847ae62017-06-27 07:06:52 -07006111 // Add some slack to account for frames dropped by the frame dropper.
6112 const int kErrorMargin = 1;
6113 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006114 kErrorMargin);
6115
6116 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006117 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006118 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006119 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006120 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006121 video_source_.IncomingCapturedFrame(
6122 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006123 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006124 ++num_frames_dropped;
6125 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006126 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006127 }
6128 }
sprang4847ae62017-06-27 07:06:52 -07006129 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006130 kErrorMargin);
6131
6132 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006133 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006134 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006135 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006136 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006137 video_source_.IncomingCapturedFrame(
6138 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006139 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006140 ++num_frames_dropped;
6141 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006142 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006143 }
6144 }
sprang4847ae62017-06-27 07:06:52 -07006145 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006146 kErrorMargin);
6147
6148 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006149 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006150 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006151 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006152 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006153 video_source_.IncomingCapturedFrame(
6154 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006155 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006156 ++num_frames_dropped;
6157 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006158 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006159 }
6160 }
6161 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6162
mflodmancc3d4422017-08-03 08:27:51 -07006163 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006164}
6165
mflodmancc3d4422017-08-03 08:27:51 -07006166TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006167 const int kFramerateFps = 5;
6168 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006169 const int kFrameWidth = 1280;
6170 const int kFrameHeight = 720;
6171
sprang4847ae62017-06-27 07:06:52 -07006172 // Reconfigure encoder with two temporal layers and screensharing, which will
6173 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006174 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006175
Henrik Boström381d1092020-05-12 18:49:07 +02006176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006177 DataRate::BitsPerSec(kTargetBitrateBps),
6178 DataRate::BitsPerSec(kTargetBitrateBps),
6179 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006180 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006181 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006182 video_source_.set_adaptation_enabled(true);
6183
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006184 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006185
6186 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006187 rtc::VideoSinkWants last_wants;
6188 do {
6189 last_wants = video_source_.sink_wants();
6190
sprangc5d62e22017-04-02 23:53:04 -07006191 // Insert frames to get a new fps estimate...
6192 for (int j = 0; j < kFramerateFps; ++j) {
6193 video_source_.IncomingCapturedFrame(
6194 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006195 if (video_source_.last_sent_width()) {
6196 sink_.WaitForEncodedFrame(timestamp_ms);
6197 }
sprangc5d62e22017-04-02 23:53:04 -07006198 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006199 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006200 }
6201 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006202 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006203 } while (video_source_.sink_wants().max_framerate_fps <
6204 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006205
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006206 EXPECT_THAT(video_source_.sink_wants(),
6207 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006208
mflodmancc3d4422017-08-03 08:27:51 -07006209 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006210}
asaperssonf7e294d2017-06-13 23:25:22 -07006211
mflodmancc3d4422017-08-03 08:27:51 -07006212TEST_F(VideoStreamEncoderTest,
6213 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006214 const int kWidth = 1280;
6215 const int kHeight = 720;
6216 const int64_t kFrameIntervalMs = 150;
6217 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006219 DataRate::BitsPerSec(kTargetBitrateBps),
6220 DataRate::BitsPerSec(kTargetBitrateBps),
6221 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006222
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006223 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006224 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006225 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006226 video_stream_encoder_->SetSource(&source,
6227 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006228 timestamp_ms += kFrameIntervalMs;
6229 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006230 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006231 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006232 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6234 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6235
6236 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006237 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006238 timestamp_ms += kFrameIntervalMs;
6239 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006240 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006241 EXPECT_THAT(source.sink_wants(),
6242 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006243 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6245 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6246
6247 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006248 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006249 timestamp_ms += kFrameIntervalMs;
6250 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006251 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006252 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006253 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6254 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6255 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6256
6257 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006258 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006259 timestamp_ms += kFrameIntervalMs;
6260 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006261 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006262 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006263 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6264 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6265 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6266
6267 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006268 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006269 timestamp_ms += kFrameIntervalMs;
6270 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006271 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006272 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006273 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6274 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6275 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6276
6277 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006278 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006279 timestamp_ms += kFrameIntervalMs;
6280 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006281 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006282 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006283 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6284 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6285 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6286
6287 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006288 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006289 timestamp_ms += kFrameIntervalMs;
6290 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006291 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006292 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006293 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6294 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6295 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6296
6297 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006298 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006299 timestamp_ms += kFrameIntervalMs;
6300 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006301 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006302 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006303 rtc::VideoSinkWants last_wants = source.sink_wants();
6304 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6305 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6306 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6307
6308 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006309 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006310 timestamp_ms += kFrameIntervalMs;
6311 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006312 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006313 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006314 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6316 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6317
Evan Shrubsole64469032020-06-11 10:45:29 +02006318 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006319 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006320 timestamp_ms += kFrameIntervalMs;
6321 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006322 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006323 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006324 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6325 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6326 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6327
6328 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006329 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006330 timestamp_ms += kFrameIntervalMs;
6331 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006332 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006333 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006334 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6335 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6336 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6337
6338 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006339 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006340 timestamp_ms += kFrameIntervalMs;
6341 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006342 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006343 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006344 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6345 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6346 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6347
6348 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006349 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006350 timestamp_ms += kFrameIntervalMs;
6351 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006352 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006353 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006354 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6355 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6356 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6357
6358 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006359 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006360 timestamp_ms += kFrameIntervalMs;
6361 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006362 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006363 EXPECT_THAT(source.sink_wants(), FpsMax());
6364 EXPECT_EQ(source.sink_wants().max_pixel_count,
6365 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006366 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6367 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6368 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6369
6370 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006371 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006372 timestamp_ms += kFrameIntervalMs;
6373 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006374 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006375 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006376 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6377 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6378 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6379
Åsa Persson30ab0152019-08-27 12:22:33 +02006380 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006381 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006382 timestamp_ms += kFrameIntervalMs;
6383 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006384 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006385 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006386 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006387 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6388 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6389 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6390
6391 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006392 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006393 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006394 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6395
mflodmancc3d4422017-08-03 08:27:51 -07006396 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006397}
6398
mflodmancc3d4422017-08-03 08:27:51 -07006399TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006400 const int kWidth = 1280;
6401 const int kHeight = 720;
6402 const int64_t kFrameIntervalMs = 150;
6403 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006404 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006405 DataRate::BitsPerSec(kTargetBitrateBps),
6406 DataRate::BitsPerSec(kTargetBitrateBps),
6407 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006408
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006409 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006410 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006411 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006412 video_stream_encoder_->SetSource(&source,
6413 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006414 timestamp_ms += kFrameIntervalMs;
6415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006416 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006417 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006418 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6419 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6420 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6421 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6422 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6423 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6424
6425 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006426 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006427 timestamp_ms += kFrameIntervalMs;
6428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006429 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006430 EXPECT_THAT(source.sink_wants(),
6431 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006432 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6434 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6435 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6436 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6437 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6438
6439 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006440 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006441 timestamp_ms += kFrameIntervalMs;
6442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006443 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006444 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006445 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6447 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6448 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6449 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6450 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6451
6452 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006453 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006454 timestamp_ms += kFrameIntervalMs;
6455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006456 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006457 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006459 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6460 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6461 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6462 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6463 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6464
Evan Shrubsole64469032020-06-11 10:45:29 +02006465 // Trigger cpu adapt up, expect no change since QP is most limited.
6466 {
6467 // Store current sink wants since we expect no change and if there is no
6468 // change then last_wants() is not updated.
6469 auto previous_sink_wants = source.sink_wants();
6470 video_stream_encoder_->TriggerCpuUnderuse();
6471 timestamp_ms += kFrameIntervalMs;
6472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6473 WaitForEncodedFrame(timestamp_ms);
6474 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6475 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6476 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6477 }
6478
6479 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6480 video_stream_encoder_->TriggerQualityHigh();
6481 timestamp_ms += kFrameIntervalMs;
6482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6483 WaitForEncodedFrame(timestamp_ms);
6484 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6485 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6487 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6488 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6489 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6490 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6491
6492 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6493 // expect increased resolution (960x540@30fps).
6494 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006495 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006496 timestamp_ms += kFrameIntervalMs;
6497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006498 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006499 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6502 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6503 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6504 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006505 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006506
Evan Shrubsole64469032020-06-11 10:45:29 +02006507 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6508 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006509 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006510 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006511 timestamp_ms += kFrameIntervalMs;
6512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006513 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006514 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006515 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006516 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6517 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6518 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6519 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6520 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006521 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006522
6523 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006524 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006525 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006526 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006527 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006528
mflodmancc3d4422017-08-03 08:27:51 -07006529 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006530}
6531
mflodmancc3d4422017-08-03 08:27:51 -07006532TEST_F(VideoStreamEncoderTest,
6533 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006534 const int kWidth = 640;
6535 const int kHeight = 360;
6536 const int kFpsLimit = 15;
6537 const int64_t kFrameIntervalMs = 150;
6538 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006539 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006540 DataRate::BitsPerSec(kTargetBitrateBps),
6541 DataRate::BitsPerSec(kTargetBitrateBps),
6542 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006543
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006544 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006545 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006546 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006547 video_stream_encoder_->SetSource(&source,
6548 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006549 timestamp_ms += kFrameIntervalMs;
6550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006551 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006552 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006553 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6555 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6556 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6557 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6558 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006565 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6569 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6570 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6571 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6572
6573 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006574 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006575 timestamp_ms += kFrameIntervalMs;
6576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006577 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006578 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006581 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6583 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6584 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6585
Evan Shrubsole64469032020-06-11 10:45:29 +02006586 // Trigger cpu adapt up, expect no change because quality is most limited.
6587 {
6588 auto previous_sink_wants = source.sink_wants();
6589 // Store current sink wants since we expect no change ind if there is no
6590 // change then last__wants() is not updated.
6591 video_stream_encoder_->TriggerCpuUnderuse();
6592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6594 WaitForEncodedFrame(timestamp_ms);
6595 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6596 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6597 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598 }
6599
6600 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6601 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006602 timestamp_ms += kFrameIntervalMs;
6603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006604 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006605 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006609 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6610 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6611 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006612
Evan Shrubsole64469032020-06-11 10:45:29 +02006613 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006614 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006615 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006616 timestamp_ms += kFrameIntervalMs;
6617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006618 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006619 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006620 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6621 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6622 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6623 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6624 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006625 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006626
6627 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006628 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006629 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006630 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006631 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006632
mflodmancc3d4422017-08-03 08:27:51 -07006633 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006634}
6635
mflodmancc3d4422017-08-03 08:27:51 -07006636TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006637 const int kFrameWidth = 1920;
6638 const int kFrameHeight = 1080;
6639 // 3/4 of 1920.
6640 const int kAdaptedFrameWidth = 1440;
6641 // 3/4 of 1080 rounded down to multiple of 4.
6642 const int kAdaptedFrameHeight = 808;
6643 const int kFramerate = 24;
6644
Henrik Boström381d1092020-05-12 18:49:07 +02006645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006646 DataRate::BitsPerSec(kTargetBitrateBps),
6647 DataRate::BitsPerSec(kTargetBitrateBps),
6648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006649 // Trigger reconfigure encoder (without resetting the entire instance).
6650 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006651 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6652 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006653 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006654 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006655 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006656 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006657 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006658 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006659
6660 video_source_.set_adaptation_enabled(true);
6661
6662 video_source_.IncomingCapturedFrame(
6663 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006664 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006665
6666 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006667 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006668 video_source_.IncomingCapturedFrame(
6669 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006670 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006671
mflodmancc3d4422017-08-03 08:27:51 -07006672 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006673}
6674
mflodmancc3d4422017-08-03 08:27:51 -07006675TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006676 const int kFrameWidth = 1280;
6677 const int kFrameHeight = 720;
6678 const int kLowFps = 2;
6679 const int kHighFps = 30;
6680
Henrik Boström381d1092020-05-12 18:49:07 +02006681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006682 DataRate::BitsPerSec(kTargetBitrateBps),
6683 DataRate::BitsPerSec(kTargetBitrateBps),
6684 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006685
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006686 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006687 max_framerate_ = kLowFps;
6688
6689 // Insert 2 seconds of 2fps video.
6690 for (int i = 0; i < kLowFps * 2; ++i) {
6691 video_source_.IncomingCapturedFrame(
6692 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6693 WaitForEncodedFrame(timestamp_ms);
6694 timestamp_ms += 1000 / kLowFps;
6695 }
6696
6697 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006699 DataRate::BitsPerSec(kTargetBitrateBps),
6700 DataRate::BitsPerSec(kTargetBitrateBps),
6701 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006702 video_source_.IncomingCapturedFrame(
6703 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6704 WaitForEncodedFrame(timestamp_ms);
6705 timestamp_ms += 1000 / kLowFps;
6706
6707 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6708
6709 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006710 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006711 const int kFrameIntervalMs = 1000 / kHighFps;
6712 max_framerate_ = kHighFps;
6713 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6714 video_source_.IncomingCapturedFrame(
6715 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6716 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6717 // be dropped if the encoder hans't been updated with the new higher target
6718 // framerate yet, causing it to overshoot the target bitrate and then
6719 // suffering the wrath of the media optimizer.
6720 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6721 timestamp_ms += kFrameIntervalMs;
6722 }
6723
6724 // Don expect correct measurement just yet, but it should be higher than
6725 // before.
6726 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6727
mflodmancc3d4422017-08-03 08:27:51 -07006728 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006729}
6730
mflodmancc3d4422017-08-03 08:27:51 -07006731TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006732 const int kFrameWidth = 1280;
6733 const int kFrameHeight = 720;
6734 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006735 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006736 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006737 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006738
Henrik Boström381d1092020-05-12 18:49:07 +02006739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006740 DataRate::BitsPerSec(kTargetBitrateBps),
6741 DataRate::BitsPerSec(kTargetBitrateBps),
6742 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006743 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006744
6745 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006746 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006747 video_source_.IncomingCapturedFrame(
6748 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6749 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006750 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006751
6752 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6754 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6755 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006756
6757 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006758 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006759 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006760
Per Kjellanderdcef6412020-10-07 15:09:05 +02006761 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006762 video_source_.IncomingCapturedFrame(
6763 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6764 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006765 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006766
mflodmancc3d4422017-08-03 08:27:51 -07006767 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006768}
ilnik6b826ef2017-06-16 06:53:48 -07006769
Niels Möller4db138e2018-04-19 09:04:13 +02006770TEST_F(VideoStreamEncoderTest,
6771 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6772 const int kFrameWidth = 1280;
6773 const int kFrameHeight = 720;
6774 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006775 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006776 DataRate::BitsPerSec(kTargetBitrateBps),
6777 DataRate::BitsPerSec(kTargetBitrateBps),
6778 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006779 video_source_.IncomingCapturedFrame(
6780 CreateFrame(1, kFrameWidth, kFrameHeight));
6781 WaitForEncodedFrame(1);
6782 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6783 .low_encode_usage_threshold_percent,
6784 default_options.low_encode_usage_threshold_percent);
6785 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6786 .high_encode_usage_threshold_percent,
6787 default_options.high_encode_usage_threshold_percent);
6788 video_stream_encoder_->Stop();
6789}
6790
6791TEST_F(VideoStreamEncoderTest,
6792 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6793 const int kFrameWidth = 1280;
6794 const int kFrameHeight = 720;
6795 CpuOveruseOptions hardware_options;
6796 hardware_options.low_encode_usage_threshold_percent = 150;
6797 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006798 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006799
Henrik Boström381d1092020-05-12 18:49:07 +02006800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006801 DataRate::BitsPerSec(kTargetBitrateBps),
6802 DataRate::BitsPerSec(kTargetBitrateBps),
6803 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006804 video_source_.IncomingCapturedFrame(
6805 CreateFrame(1, kFrameWidth, kFrameHeight));
6806 WaitForEncodedFrame(1);
6807 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6808 .low_encode_usage_threshold_percent,
6809 hardware_options.low_encode_usage_threshold_percent);
6810 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6811 .high_encode_usage_threshold_percent,
6812 hardware_options.high_encode_usage_threshold_percent);
6813 video_stream_encoder_->Stop();
6814}
6815
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006816TEST_F(VideoStreamEncoderTest,
6817 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6818 const int kFrameWidth = 1280;
6819 const int kFrameHeight = 720;
6820
6821 const CpuOveruseOptions default_options;
6822 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6823 DataRate::BitsPerSec(kTargetBitrateBps),
6824 DataRate::BitsPerSec(kTargetBitrateBps),
6825 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6826 video_source_.IncomingCapturedFrame(
6827 CreateFrame(1, kFrameWidth, kFrameHeight));
6828 WaitForEncodedFrame(1);
6829 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6830 .low_encode_usage_threshold_percent,
6831 default_options.low_encode_usage_threshold_percent);
6832 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6833 .high_encode_usage_threshold_percent,
6834 default_options.high_encode_usage_threshold_percent);
6835
6836 CpuOveruseOptions hardware_options;
6837 hardware_options.low_encode_usage_threshold_percent = 150;
6838 hardware_options.high_encode_usage_threshold_percent = 200;
6839 fake_encoder_.SetIsHardwareAccelerated(true);
6840
6841 video_source_.IncomingCapturedFrame(
6842 CreateFrame(2, kFrameWidth, kFrameHeight));
6843 WaitForEncodedFrame(2);
6844
6845 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6846 .low_encode_usage_threshold_percent,
6847 hardware_options.low_encode_usage_threshold_percent);
6848 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6849 .high_encode_usage_threshold_percent,
6850 hardware_options.high_encode_usage_threshold_percent);
6851
6852 video_stream_encoder_->Stop();
6853}
6854
Niels Möller6bb5ab92019-01-11 11:11:10 +01006855TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6856 const int kFrameWidth = 320;
6857 const int kFrameHeight = 240;
6858 const int kFps = 30;
6859 const int kTargetBitrateBps = 120000;
6860 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6861
Henrik Boström381d1092020-05-12 18:49:07 +02006862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006863 DataRate::BitsPerSec(kTargetBitrateBps),
6864 DataRate::BitsPerSec(kTargetBitrateBps),
6865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006866
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006867 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006868 max_framerate_ = kFps;
6869
6870 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6871 fake_encoder_.SimulateOvershoot(1.0);
6872 int num_dropped = 0;
6873 for (int i = 0; i < kNumFramesInRun; ++i) {
6874 video_source_.IncomingCapturedFrame(
6875 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6876 // Wait up to two frame durations for a frame to arrive.
6877 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6878 ++num_dropped;
6879 }
6880 timestamp_ms += 1000 / kFps;
6881 }
6882
Erik Språnga8d48ab2019-02-08 14:17:40 +01006883 // Framerate should be measured to be near the expected target rate.
6884 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6885
6886 // Frame drops should be within 5% of expected 0%.
6887 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006888
6889 // Make encoder produce frames at double the expected bitrate during 3 seconds
6890 // of video, verify number of drops. Rate needs to be slightly changed in
6891 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006892 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006893 const RateControlSettings trials =
6894 RateControlSettings::ParseFromFieldTrials();
6895 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006896 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006897 // frame dropping since the adjuter will try to just lower the target
6898 // bitrate rather than drop frames. If network headroom can be used, it
6899 // doesn't push back as hard so we don't need quite as much overshoot.
6900 // These numbers are unfortunately a bit magical but there's not trivial
6901 // way to algebraically infer them.
6902 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6903 overshoot_factor = 2.4;
6904 } else {
6905 overshoot_factor = 4.0;
6906 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006907 }
6908 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006910 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6911 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6912 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006913 num_dropped = 0;
6914 for (int i = 0; i < kNumFramesInRun; ++i) {
6915 video_source_.IncomingCapturedFrame(
6916 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6917 // Wait up to two frame durations for a frame to arrive.
6918 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6919 ++num_dropped;
6920 }
6921 timestamp_ms += 1000 / kFps;
6922 }
6923
Henrik Boström381d1092020-05-12 18:49:07 +02006924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006925 DataRate::BitsPerSec(kTargetBitrateBps),
6926 DataRate::BitsPerSec(kTargetBitrateBps),
6927 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006928
6929 // Target framerate should be still be near the expected target, despite
6930 // the frame drops.
6931 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6932
6933 // Frame drops should be within 5% of expected 50%.
6934 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006935
6936 video_stream_encoder_->Stop();
6937}
6938
6939TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6940 const int kFrameWidth = 320;
6941 const int kFrameHeight = 240;
6942 const int kActualInputFps = 24;
6943 const int kTargetBitrateBps = 120000;
6944
6945 ASSERT_GT(max_framerate_, kActualInputFps);
6946
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006947 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006948 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006949 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006950 DataRate::BitsPerSec(kTargetBitrateBps),
6951 DataRate::BitsPerSec(kTargetBitrateBps),
6952 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006953
6954 // Insert 3 seconds of video, with an input fps lower than configured max.
6955 for (int i = 0; i < kActualInputFps * 3; ++i) {
6956 video_source_.IncomingCapturedFrame(
6957 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6958 // Wait up to two frame durations for a frame to arrive.
6959 WaitForEncodedFrame(timestamp_ms);
6960 timestamp_ms += 1000 / kActualInputFps;
6961 }
6962
6963 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6964
6965 video_stream_encoder_->Stop();
6966}
6967
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006968TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006969 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006970 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006971 DataRate::BitsPerSec(kTargetBitrateBps),
6972 DataRate::BitsPerSec(kTargetBitrateBps),
6973 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006974
6975 fake_encoder_.BlockNextEncode();
6976 video_source_.IncomingCapturedFrame(
6977 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6978 WaitForEncodedFrame(1);
6979 // On the very first frame full update should be forced.
6980 rect = fake_encoder_.GetLastUpdateRect();
6981 EXPECT_EQ(rect.offset_x, 0);
6982 EXPECT_EQ(rect.offset_y, 0);
6983 EXPECT_EQ(rect.height, codec_height_);
6984 EXPECT_EQ(rect.width, codec_width_);
6985 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6986 // call to ContinueEncode.
6987 video_source_.IncomingCapturedFrame(
6988 CreateFrameWithUpdatedPixel(2, nullptr, 1));
6989 ExpectDroppedFrame();
6990 video_source_.IncomingCapturedFrame(
6991 CreateFrameWithUpdatedPixel(3, nullptr, 10));
6992 ExpectDroppedFrame();
6993 fake_encoder_.ContinueEncode();
6994 WaitForEncodedFrame(3);
6995 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
6996 rect = fake_encoder_.GetLastUpdateRect();
6997 EXPECT_EQ(rect.offset_x, 1);
6998 EXPECT_EQ(rect.offset_y, 0);
6999 EXPECT_EQ(rect.width, 10);
7000 EXPECT_EQ(rect.height, 1);
7001
7002 video_source_.IncomingCapturedFrame(
7003 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7004 WaitForEncodedFrame(4);
7005 // Previous frame was encoded, so no accumulation should happen.
7006 rect = fake_encoder_.GetLastUpdateRect();
7007 EXPECT_EQ(rect.offset_x, 0);
7008 EXPECT_EQ(rect.offset_y, 0);
7009 EXPECT_EQ(rect.width, 1);
7010 EXPECT_EQ(rect.height, 1);
7011
7012 video_stream_encoder_->Stop();
7013}
7014
Erik Språngd7329ca2019-02-21 21:19:53 +01007015TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007016 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007017 DataRate::BitsPerSec(kTargetBitrateBps),
7018 DataRate::BitsPerSec(kTargetBitrateBps),
7019 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007020
7021 // First frame is always keyframe.
7022 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7023 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007024 EXPECT_THAT(
7025 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007026 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007027
7028 // Insert delta frame.
7029 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7030 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007031 EXPECT_THAT(
7032 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007033 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007034
7035 // Request next frame be a key-frame.
7036 video_stream_encoder_->SendKeyFrame();
7037 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7038 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007039 EXPECT_THAT(
7040 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007041 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007042
7043 video_stream_encoder_->Stop();
7044}
7045
7046TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7047 // Setup simulcast with three streams.
7048 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007050 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7051 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7052 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007053 // Wait for all three layers before triggering event.
7054 sink_.SetNumExpectedLayers(3);
7055
7056 // First frame is always keyframe.
7057 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7058 WaitForEncodedFrame(1);
7059 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007060 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7061 VideoFrameType::kVideoFrameKey,
7062 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007063
7064 // Insert delta frame.
7065 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7066 WaitForEncodedFrame(2);
7067 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007068 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7069 VideoFrameType::kVideoFrameDelta,
7070 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007071
7072 // Request next frame be a key-frame.
7073 // Only first stream is configured to produce key-frame.
7074 video_stream_encoder_->SendKeyFrame();
7075 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7076 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007077
7078 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7079 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007080 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007081 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007082 VideoFrameType::kVideoFrameKey,
7083 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007084
7085 video_stream_encoder_->Stop();
7086}
7087
7088TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7089 // Configure internal source factory and setup test again.
7090 encoder_factory_.SetHasInternalSource(true);
7091 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007092 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007093 DataRate::BitsPerSec(kTargetBitrateBps),
7094 DataRate::BitsPerSec(kTargetBitrateBps),
7095 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007096
7097 // Call encoder directly, simulating internal source where encoded frame
7098 // callback in VideoStreamEncoder is called despite no OnFrame().
7099 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7100 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007101 EXPECT_THAT(
7102 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007103 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007104
Niels Möller8f7ce222019-03-21 15:43:58 +01007105 const std::vector<VideoFrameType> kDeltaFrame = {
7106 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007107 // Need to set timestamp manually since manually for injected frame.
7108 VideoFrame frame = CreateFrame(101, nullptr);
7109 frame.set_timestamp(101);
7110 fake_encoder_.InjectFrame(frame, false);
7111 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007112 EXPECT_THAT(
7113 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007114 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007115
7116 // Request key-frame. The forces a dummy frame down into the encoder.
7117 fake_encoder_.ExpectNullFrame();
7118 video_stream_encoder_->SendKeyFrame();
7119 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007120 EXPECT_THAT(
7121 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007122 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007123
7124 video_stream_encoder_->Stop();
7125}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007126
7127TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7128 // Configure internal source factory and setup test again.
7129 encoder_factory_.SetHasInternalSource(true);
7130 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007132 DataRate::BitsPerSec(kTargetBitrateBps),
7133 DataRate::BitsPerSec(kTargetBitrateBps),
7134 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007135
7136 int64_t timestamp = 1;
7137 EncodedImage image;
Sergey Silkin727d2af2021-03-11 17:05:44 +00007138 image.SetEncodedData(
7139 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01007140 image.capture_time_ms_ = ++timestamp;
7141 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7142 const int64_t kEncodeFinishDelayMs = 10;
7143 image.timing_.encode_start_ms = timestamp;
7144 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin727d2af2021-03-11 17:05:44 +00007145 fake_encoder_.InjectEncodedImage(image);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007146 // Wait for frame without incrementing clock.
7147 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7148 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7149 // capture timestamp should be kEncodeFinishDelayMs in the past.
7150 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007151 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007152
7153 video_stream_encoder_->Stop();
7154}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007155
7156TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007157 // SPS contains VUI with restrictions on the maximum number of reordered
7158 // pictures, there is no need to rewrite the bitstream to enable faster
7159 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007160 ResetEncoder("H264", 1, 1, 1, false);
7161
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7163 DataRate::BitsPerSec(kTargetBitrateBps),
7164 DataRate::BitsPerSec(kTargetBitrateBps),
7165 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7166 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007167
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007168 fake_encoder_.SetEncodedImageData(
7169 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007170
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007171 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7172 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007173
7174 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7175 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007176
7177 video_stream_encoder_->Stop();
7178}
7179
7180TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007181 // SPS does not contain VUI, the bitstream is will be rewritten with added
7182 // VUI with restrictions on the maximum number of reordered pictures to
7183 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007184 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7185 0x00, 0x00, 0x03, 0x03, 0xF4,
7186 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007187 ResetEncoder("H264", 1, 1, 1, false);
7188
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7190 DataRate::BitsPerSec(kTargetBitrateBps),
7191 DataRate::BitsPerSec(kTargetBitrateBps),
7192 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7193 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007194
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007195 fake_encoder_.SetEncodedImageData(
7196 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007197
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007198 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7199 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007200
7201 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7202 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007203
7204 video_stream_encoder_->Stop();
7205}
7206
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007207TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7208 const int kFrameWidth = 1280;
7209 const int kFrameHeight = 720;
7210 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7211
Henrik Boström381d1092020-05-12 18:49:07 +02007212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007213 DataRate::BitsPerSec(kTargetBitrateBps),
7214 DataRate::BitsPerSec(kTargetBitrateBps),
7215 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007216 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7217
7218 // Insert a first video frame. It should be dropped because of downscale in
7219 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007220 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007221 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7222 frame.set_rotation(kVideoRotation_270);
7223 video_source_.IncomingCapturedFrame(frame);
7224
7225 ExpectDroppedFrame();
7226
7227 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007228 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007229 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7230 frame.set_rotation(kVideoRotation_90);
7231 video_source_.IncomingCapturedFrame(frame);
7232
7233 WaitForEncodedFrame(timestamp_ms);
7234 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7235
7236 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007237 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007238 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7239 frame.set_rotation(kVideoRotation_180);
7240 video_source_.IncomingCapturedFrame(frame);
7241
7242 WaitForEncodedFrame(timestamp_ms);
7243 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7244
7245 video_stream_encoder_->Stop();
7246}
7247
Erik Språng5056af02019-09-02 15:53:11 +02007248TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7249 const int kFrameWidth = 320;
7250 const int kFrameHeight = 180;
7251
7252 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007254 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7255 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7256 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007257 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007258 /*rtt_ms=*/0,
7259 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007260
7261 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007262 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007263 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7264 frame.set_rotation(kVideoRotation_270);
7265 video_source_.IncomingCapturedFrame(frame);
7266 WaitForEncodedFrame(timestamp_ms);
7267
7268 // Set a target rate below the minimum allowed by the codec settings.
7269 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007270 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7271 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007273 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007274 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007275 /*link_allocation=*/target_rate,
7276 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007277 /*rtt_ms=*/0,
7278 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007279 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7280
7281 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7282 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7283 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007284 DataRate allocation_sum =
7285 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007286 EXPECT_EQ(min_rate, allocation_sum);
7287 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7288
7289 video_stream_encoder_->Stop();
7290}
7291
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007292TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007294 DataRate::BitsPerSec(kTargetBitrateBps),
7295 DataRate::BitsPerSec(kTargetBitrateBps),
7296 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007297 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007298 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007299 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7300 WaitForEncodedFrame(1);
7301
7302 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7303 ASSERT_TRUE(prev_rate_settings.has_value());
7304 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7305 kDefaultFramerate);
7306
7307 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7308 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7309 timestamp_ms += 1000 / kDefaultFramerate;
7310 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7311 WaitForEncodedFrame(timestamp_ms);
7312 }
7313 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7314 kDefaultFramerate);
7315 // Capture larger frame to trigger a reconfigure.
7316 codec_height_ *= 2;
7317 codec_width_ *= 2;
7318 timestamp_ms += 1000 / kDefaultFramerate;
7319 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7320 WaitForEncodedFrame(timestamp_ms);
7321
7322 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7323 auto current_rate_settings =
7324 fake_encoder_.GetAndResetLastRateControlSettings();
7325 // Ensure we have actually reconfigured twice
7326 // The rate settings should have been set again even though
7327 // they haven't changed.
7328 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007329 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007330
7331 video_stream_encoder_->Stop();
7332}
7333
philipeld9cc8c02019-09-16 14:53:40 +02007334struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007335 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7336 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7337 MOCK_METHOD(void,
7338 RequestEncoderSwitch,
7339 (const webrtc::SdpVideoFormat& format),
7340 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007341};
7342
philipel9b058032020-02-10 11:30:00 +01007343TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7344 constexpr int kDontCare = 100;
7345 StrictMock<MockEncoderSelector> encoder_selector;
7346 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7347 &fake_encoder_, &encoder_selector);
7348 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7349
7350 // Reset encoder for new configuration to take effect.
7351 ConfigureEncoder(video_encoder_config_.Copy());
7352
7353 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7354
7355 video_source_.IncomingCapturedFrame(
7356 CreateFrame(kDontCare, kDontCare, kDontCare));
7357 video_stream_encoder_->Stop();
7358
7359 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7360 // to it's factory, so in order for the encoder instance in the
7361 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7362 // reset the |video_stream_encoder_| here.
7363 video_stream_encoder_.reset();
7364}
7365
7366TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7367 constexpr int kDontCare = 100;
7368
7369 NiceMock<MockEncoderSelector> encoder_selector;
7370 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7371 video_send_config_.encoder_settings.encoder_switch_request_callback =
7372 &switch_callback;
7373 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7374 &fake_encoder_, &encoder_selector);
7375 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7376
7377 // Reset encoder for new configuration to take effect.
7378 ConfigureEncoder(video_encoder_config_.Copy());
7379
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007380 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007381 .WillByDefault(Return(SdpVideoFormat("AV1")));
7382 EXPECT_CALL(switch_callback,
7383 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7384 Field(&SdpVideoFormat::name, "AV1"))));
7385
Henrik Boström381d1092020-05-12 18:49:07 +02007386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007387 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7388 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7389 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007390 /*fraction_lost=*/0,
7391 /*rtt_ms=*/0,
7392 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007393 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007394
7395 video_stream_encoder_->Stop();
7396}
7397
7398TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7399 constexpr int kSufficientBitrateToNotDrop = 1000;
7400 constexpr int kDontCare = 100;
7401
7402 NiceMock<MockVideoEncoder> video_encoder;
7403 NiceMock<MockEncoderSelector> encoder_selector;
7404 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7405 video_send_config_.encoder_settings.encoder_switch_request_callback =
7406 &switch_callback;
7407 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7408 &video_encoder, &encoder_selector);
7409 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7410
7411 // Reset encoder for new configuration to take effect.
7412 ConfigureEncoder(video_encoder_config_.Copy());
7413
7414 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7415 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7416 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007418 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7419 /*stable_target_bitrate=*/
7420 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7421 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007422 /*fraction_lost=*/0,
7423 /*rtt_ms=*/0,
7424 /*cwnd_reduce_ratio=*/0);
7425
7426 ON_CALL(video_encoder, Encode(_, _))
7427 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7428 ON_CALL(encoder_selector, OnEncoderBroken())
7429 .WillByDefault(Return(SdpVideoFormat("AV2")));
7430
7431 rtc::Event encode_attempted;
7432 EXPECT_CALL(switch_callback,
7433 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7434 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7435 EXPECT_EQ(format.name, "AV2");
7436 encode_attempted.Set();
7437 });
7438
7439 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7440 encode_attempted.Wait(3000);
7441
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007442 AdvanceTime(TimeDelta::Millis(0));
7443
philipel9b058032020-02-10 11:30:00 +01007444 video_stream_encoder_->Stop();
7445
7446 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7447 // to it's factory, so in order for the encoder instance in the
7448 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7449 // reset the |video_stream_encoder_| here.
7450 video_stream_encoder_.reset();
7451}
7452
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007453TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007454 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007455 const int kFrameWidth = 320;
7456 const int kFrameHeight = 180;
7457
7458 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007459 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007460 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007461 /*target_bitrate=*/rate,
7462 /*stable_target_bitrate=*/rate,
7463 /*link_allocation=*/rate,
7464 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007465 /*rtt_ms=*/0,
7466 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007467
7468 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007469 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007470 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7471 frame.set_rotation(kVideoRotation_270);
7472 video_source_.IncomingCapturedFrame(frame);
7473 WaitForEncodedFrame(timestamp_ms);
7474 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7475
7476 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007477 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007479 /*target_bitrate=*/new_stable_rate,
7480 /*stable_target_bitrate=*/new_stable_rate,
7481 /*link_allocation=*/rate,
7482 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007483 /*rtt_ms=*/0,
7484 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007485 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7486 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7487 video_stream_encoder_->Stop();
7488}
7489
7490TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007491 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007492 const int kFrameWidth = 320;
7493 const int kFrameHeight = 180;
7494
7495 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007496 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007498 /*target_bitrate=*/rate,
7499 /*stable_target_bitrate=*/rate,
7500 /*link_allocation=*/rate,
7501 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007502 /*rtt_ms=*/0,
7503 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007504
7505 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007506 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007507 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7508 frame.set_rotation(kVideoRotation_270);
7509 video_source_.IncomingCapturedFrame(frame);
7510 WaitForEncodedFrame(timestamp_ms);
7511 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7512
7513 // Set a higher target rate without changing the link_allocation. Should not
7514 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007515 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007517 /*target_bitrate=*/rate,
7518 /*stable_target_bitrate=*/new_stable_rate,
7519 /*link_allocation=*/rate,
7520 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007521 /*rtt_ms=*/0,
7522 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007523 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7524 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7525 video_stream_encoder_->Stop();
7526}
7527
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007528TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7529 test::ScopedFieldTrials field_trials(
7530 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7531 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7532 const int kFramerateFps = 30;
7533 const int kWidth = 1920;
7534 const int kHeight = 1080;
7535 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7536 // Works on screenshare mode.
7537 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7538 // We rely on the automatic resolution adaptation, but we handle framerate
7539 // adaptation manually by mocking the stats proxy.
7540 video_source_.set_adaptation_enabled(true);
7541
7542 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007544 DataRate::BitsPerSec(kTargetBitrateBps),
7545 DataRate::BitsPerSec(kTargetBitrateBps),
7546 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007547 video_stream_encoder_->SetSource(&video_source_,
7548 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007549 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007550
7551 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7552 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7553
7554 // Pass enough frames with the full update to trigger animation detection.
7555 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007556 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007557 frame.set_ntp_time_ms(timestamp_ms);
7558 frame.set_timestamp_us(timestamp_ms * 1000);
7559 video_source_.IncomingCapturedFrame(frame);
7560 WaitForEncodedFrame(timestamp_ms);
7561 }
7562
7563 // Resolution should be limited.
7564 rtc::VideoSinkWants expected;
7565 expected.max_framerate_fps = kFramerateFps;
7566 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007567 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007568
7569 // Pass one frame with no known update.
7570 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007571 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007572 frame.set_ntp_time_ms(timestamp_ms);
7573 frame.set_timestamp_us(timestamp_ms * 1000);
7574 frame.clear_update_rect();
7575
7576 video_source_.IncomingCapturedFrame(frame);
7577 WaitForEncodedFrame(timestamp_ms);
7578
7579 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007580 EXPECT_THAT(video_source_.sink_wants(),
7581 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007582
7583 video_stream_encoder_->Stop();
7584}
7585
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007586TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7587 const int kWidth = 720; // 540p adapted down.
7588 const int kHeight = 405;
7589 const int kNumFrames = 3;
7590 // Works on screenshare mode.
7591 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7592 /*num_spatial_layers=*/2, /*screenshare=*/true);
7593
7594 video_source_.set_adaptation_enabled(true);
7595
7596 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7597 DataRate::BitsPerSec(kTargetBitrateBps),
7598 DataRate::BitsPerSec(kTargetBitrateBps),
7599 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7600
7601 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7602
7603 // Pass enough frames with the full update to trigger animation detection.
7604 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007605 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007606 frame.set_ntp_time_ms(timestamp_ms);
7607 frame.set_timestamp_us(timestamp_ms * 1000);
7608 video_source_.IncomingCapturedFrame(frame);
7609 WaitForEncodedFrame(timestamp_ms);
7610 }
7611
7612 video_stream_encoder_->Stop();
7613}
7614
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007615TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7616 const float downscale_factors[] = {4.0, 2.0, 1.0};
7617 const int number_layers =
7618 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7619 VideoEncoderConfig config;
7620 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7621 for (int i = 0; i < number_layers; ++i) {
7622 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7623 config.simulcast_layers[i].active = true;
7624 }
7625 config.video_stream_factory =
7626 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7627 "VP8", /*max qp*/ 56, /*screencast*/ false,
7628 /*screenshare enabled*/ false);
7629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7630 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7631 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7632 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7633
7634 // First initialization.
7635 // Encoder should be initialized. Next frame should be key frame.
7636 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7637 sink_.SetNumExpectedLayers(number_layers);
7638 int64_t timestamp_ms = kFrameIntervalMs;
7639 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7640 WaitForEncodedFrame(timestamp_ms);
7641 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7642 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7643 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7644 VideoFrameType::kVideoFrameKey,
7645 VideoFrameType::kVideoFrameKey}));
7646
7647 // Disable top layer.
7648 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7649 config.simulcast_layers[number_layers - 1].active = false;
7650 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7651 sink_.SetNumExpectedLayers(number_layers - 1);
7652 timestamp_ms += kFrameIntervalMs;
7653 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7654 WaitForEncodedFrame(timestamp_ms);
7655 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7656 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7657 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7658 VideoFrameType::kVideoFrameDelta,
7659 VideoFrameType::kVideoFrameDelta}));
7660
7661 // Re-enable top layer.
7662 // Encoder should be re-initialized. Next frame should be key frame.
7663 config.simulcast_layers[number_layers - 1].active = true;
7664 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7665 sink_.SetNumExpectedLayers(number_layers);
7666 timestamp_ms += kFrameIntervalMs;
7667 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7668 WaitForEncodedFrame(timestamp_ms);
7669 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7670 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7671 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7672 VideoFrameType::kVideoFrameKey,
7673 VideoFrameType::kVideoFrameKey}));
7674
7675 // Top layer max rate change.
7676 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7677 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7678 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7679 sink_.SetNumExpectedLayers(number_layers);
7680 timestamp_ms += kFrameIntervalMs;
7681 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7682 WaitForEncodedFrame(timestamp_ms);
7683 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7684 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7685 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7686 VideoFrameType::kVideoFrameDelta,
7687 VideoFrameType::kVideoFrameDelta}));
7688
7689 // Top layer resolution change.
7690 // Encoder should be re-initialized. Next frame should be key frame.
7691 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7692 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7693 sink_.SetNumExpectedLayers(number_layers);
7694 timestamp_ms += kFrameIntervalMs;
7695 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7696 WaitForEncodedFrame(timestamp_ms);
7697 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7698 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7699 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7700 VideoFrameType::kVideoFrameKey,
7701 VideoFrameType::kVideoFrameKey}));
7702 video_stream_encoder_->Stop();
7703}
7704
Henrik Boström1124ed12021-02-25 10:30:39 +01007705TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7706 const int kFrameWidth = 1280;
7707 const int kFrameHeight = 720;
7708
7709 SetUp();
7710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7711 DataRate::BitsPerSec(kTargetBitrateBps),
7712 DataRate::BitsPerSec(kTargetBitrateBps),
7713 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7714
7715 // Capturing a frame should reconfigure the encoder and expose the encoder
7716 // resolution, which is the same as the input frame.
7717 int64_t timestamp_ms = kFrameIntervalMs;
7718 video_source_.IncomingCapturedFrame(
7719 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7720 WaitForEncodedFrame(timestamp_ms);
7721 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7722 EXPECT_THAT(video_source_.sink_wants().resolutions,
7723 ::testing::ElementsAreArray(
7724 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7725
7726 video_stream_encoder_->Stop();
7727}
7728
7729TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7730 // Pick downscale factors such that we never encode at full resolution - this
7731 // is an interesting use case. The frame resolution influences the encoder
7732 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7733 // encoder should not ask for the frame resolution. This allows video frames
7734 // to have the appearence of one resolution but optimize its internal buffers
7735 // for what is actually encoded.
7736 const size_t kNumSimulcastLayers = 3u;
7737 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7738 const int kFrameWidth = 1280;
7739 const int kFrameHeight = 720;
7740 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7741 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7742 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7743 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7744 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7745 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7746
7747 VideoEncoderConfig config;
7748 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7749 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7750 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7751 config.simulcast_layers[i].active = true;
7752 }
7753 config.video_stream_factory =
7754 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7755 "VP8", /*max qp*/ 56, /*screencast*/ false,
7756 /*screenshare enabled*/ false);
7757 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7758 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7759 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7760 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7761
7762 // Capture a frame with all layers active.
7763 int64_t timestamp_ms = kFrameIntervalMs;
7764 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7765 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7766 video_source_.IncomingCapturedFrame(
7767 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7768 WaitForEncodedFrame(timestamp_ms);
7769 // Expect encoded resolutions to match the expected simulcast layers.
7770 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7771 EXPECT_THAT(
7772 video_source_.sink_wants().resolutions,
7773 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7774
7775 // Capture a frame with one of the layers inactive.
7776 timestamp_ms += kFrameIntervalMs;
7777 config.simulcast_layers[2].active = false;
7778 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7779 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7780 video_source_.IncomingCapturedFrame(
7781 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7782 WaitForEncodedFrame(timestamp_ms);
7783
7784 // Expect encoded resolutions to match the expected simulcast layers.
7785 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7786 EXPECT_THAT(video_source_.sink_wants().resolutions,
7787 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7788
7789 // Capture a frame with all but one layer turned off.
7790 timestamp_ms += kFrameIntervalMs;
7791 config.simulcast_layers[1].active = false;
7792 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7793 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7794 video_source_.IncomingCapturedFrame(
7795 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7796 WaitForEncodedFrame(timestamp_ms);
7797
7798 // Expect encoded resolutions to match the expected simulcast layers.
7799 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7800 EXPECT_THAT(video_source_.sink_wants().resolutions,
7801 ::testing::ElementsAreArray({kLayer0Size}));
7802
7803 video_stream_encoder_->Stop();
7804}
7805
perkj26091b12016-09-01 01:17:40 -07007806} // namespace webrtc