blob: 8065f6287c6cd346c4b0f9a029cd46e6f70e6f11 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010040#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020041#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080043#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020044#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010045#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020046#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020047#include "test/encoder_settings.h"
48#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020049#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010050#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "test/gmock.h"
52#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020053#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020054#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070056
57namespace webrtc {
58
sprang57c2fff2017-01-16 06:24:02 -080059using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020060using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020061using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020062using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020063using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020064using ::testing::Ge;
65using ::testing::Gt;
66using ::testing::Le;
67using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010068using ::testing::Matcher;
69using ::testing::NiceMock;
70using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010071using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020072using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080073
perkj803d97f2016-11-01 11:45:46 -070074namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010076const int kQpLow = 1;
77const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020078const int kMinFramerateFps = 2;
79const int kMinBalancedFramerateFps = 7;
80const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080081const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010082const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020083const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010084const uint32_t kSimulcastTargetBitrateBps = 3150000;
85const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080086const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070087const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020089const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020090const VideoEncoder::ResolutionBitrateLimits
91 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
92const VideoEncoder::ResolutionBitrateLimits
93 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080094
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020095uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
96 0x00, 0x00, 0x03, 0x03, 0xF4,
97 0x05, 0x03, 0xC7, 0xE0, 0x1B,
98 0x41, 0x10, 0x8D, 0x00};
99
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100100const uint8_t kCodedFrameVp8Qp25[] = {
101 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
102 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
103 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
104
perkj803d97f2016-11-01 11:45:46 -0700105class TestBuffer : public webrtc::I420Buffer {
106 public:
107 TestBuffer(rtc::Event* event, int width, int height)
108 : I420Buffer(width, height), event_(event) {}
109
110 private:
111 friend class rtc::RefCountedObject<TestBuffer>;
112 ~TestBuffer() override {
113 if (event_)
114 event_->Set();
115 }
116 rtc::Event* const event_;
117};
118
Noah Richards51db4212019-06-12 06:59:12 -0700119// A fake native buffer that can't be converted to I420.
120class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
121 public:
122 FakeNativeBuffer(rtc::Event* event, int width, int height)
123 : event_(event), width_(width), height_(height) {}
124 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
125 int width() const override { return width_; }
126 int height() const override { return height_; }
127 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
128 return nullptr;
129 }
130
131 private:
132 friend class rtc::RefCountedObject<FakeNativeBuffer>;
133 ~FakeNativeBuffer() override {
134 if (event_)
135 event_->Set();
136 }
137 rtc::Event* const event_;
138 const int width_;
139 const int height_;
140};
141
Evan Shrubsole895556e2020-10-05 09:15:13 +0200142// A fake native buffer that is backed by an NV12 buffer.
143class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
144 public:
145 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
146 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
147
148 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
149 int width() const override { return nv12_buffer_->width(); }
150 int height() const override { return nv12_buffer_->height(); }
151 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
152 return nv12_buffer_->ToI420();
153 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200154 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
155 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
156 if (absl::c_find(types, Type::kNV12) != types.end()) {
157 return nv12_buffer_;
158 }
159 return nullptr;
160 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200161 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
162
163 private:
164 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
165 ~FakeNV12NativeBuffer() override {
166 if (event_)
167 event_->Set();
168 }
169 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
170 rtc::Event* const event_;
171};
172
Niels Möller7dc26b72017-12-06 10:27:48 +0100173class CpuOveruseDetectorProxy : public OveruseFrameDetector {
174 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200175 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
176 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200177 last_target_framerate_fps_(-1),
178 framerate_updated_event_(true /* manual_reset */,
179 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100180 virtual ~CpuOveruseDetectorProxy() {}
181
182 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200183 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100184 last_target_framerate_fps_ = framerate_fps;
185 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200186 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100187 }
188
189 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200190 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100191 return last_target_framerate_fps_;
192 }
193
Niels Möller4db138e2018-04-19 09:04:13 +0200194 CpuOveruseOptions GetOptions() { return options_; }
195
Henrik Boström381d1092020-05-12 18:49:07 +0200196 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
197
Niels Möller7dc26b72017-12-06 10:27:48 +0100198 private:
Markus Handella3765182020-07-08 13:13:32 +0200199 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100200 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200201 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100202};
203
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200204class FakeVideoSourceRestrictionsListener
205 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200206 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200207 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200208 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200209 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200210 RTC_DCHECK(was_restrictions_updated_);
211 }
212
213 rtc::Event* restrictions_updated_event() {
214 return &restrictions_updated_event_;
215 }
216
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200217 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200218 void OnVideoSourceRestrictionsUpdated(
219 VideoSourceRestrictions restrictions,
220 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200221 rtc::scoped_refptr<Resource> reason,
222 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200223 was_restrictions_updated_ = true;
224 restrictions_updated_event_.Set();
225 }
226
227 private:
228 bool was_restrictions_updated_;
229 rtc::Event restrictions_updated_event_;
230};
231
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200232auto WantsFps(Matcher<int> fps_matcher) {
233 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
234 fps_matcher);
235}
236
237auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
238 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
239 AllOf(max_pixel_matcher, Gt(0)));
240}
241
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200242auto ResolutionMax() {
243 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
246 Eq(absl::nullopt)));
247}
248
249auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200250 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200251}
252
253auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200254 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200255}
256
257auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200258 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200259}
260
261auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200262 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200263}
264
265auto FpsMaxResolutionMax() {
266 return AllOf(FpsMax(), ResolutionMax());
267}
268
269auto UnlimitedSinkWants() {
270 return AllOf(FpsUnlimited(), ResolutionMax());
271}
272
273auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
274 Matcher<int> fps_range_matcher;
275
276 if (last_frame_pixels <= 320 * 240) {
277 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200278 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200279 fps_range_matcher = AllOf(Ge(10), Le(15));
280 } else if (last_frame_pixels <= 640 * 480) {
281 fps_range_matcher = Ge(15);
282 } else {
283 fps_range_matcher = Eq(kDefaultFramerate);
284 }
285 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
286 fps_range_matcher);
287}
288
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200289auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
290 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
291 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
292}
293
294auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
295 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
296}
297
298auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
299 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
300}
301
302auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
303 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
304 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
305}
306
307auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
308 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
309 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
310}
311
312auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
313 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
314 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
315}
316
317auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
318 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
319 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
320}
321
mflodmancc3d4422017-08-03 08:27:51 -0700322class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700323 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200324 VideoStreamEncoderUnderTest(TimeController* time_controller,
325 TaskQueueFactory* task_queue_factory,
326 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100327 const VideoStreamEncoderSettings& settings,
328 VideoStreamEncoder::BitrateAllocationCallbackType
329 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200330 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100331 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200332 stats_proxy,
333 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200334 std::unique_ptr<OveruseFrameDetector>(
335 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100336 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100337 task_queue_factory,
338 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200339 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200340 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200341 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200342 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200343 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200344 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200345 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200346 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100347 }
perkj803d97f2016-11-01 11:45:46 -0700348
Henrik Boström381d1092020-05-12 18:49:07 +0200349 void SetSourceAndWaitForRestrictionsUpdated(
350 rtc::VideoSourceInterface<VideoFrame>* source,
351 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200352 FakeVideoSourceRestrictionsListener listener;
353 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200354 SetSource(source, degradation_preference);
355 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200356 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200357 }
358
359 void SetSourceAndWaitForFramerateUpdated(
360 rtc::VideoSourceInterface<VideoFrame>* source,
361 const DegradationPreference& degradation_preference) {
362 overuse_detector_proxy_->framerate_updated_event()->Reset();
363 SetSource(source, degradation_preference);
364 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
365 }
366
367 void OnBitrateUpdatedAndWaitForManagedResources(
368 DataRate target_bitrate,
369 DataRate stable_target_bitrate,
370 DataRate link_allocation,
371 uint8_t fraction_lost,
372 int64_t round_trip_time_ms,
373 double cwnd_reduce_ratio) {
374 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
375 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
376 // Bitrate is updated on the encoder queue.
377 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200378 }
379
kthelgason2fc52542017-03-03 00:24:41 -0800380 // This is used as a synchronisation mechanism, to make sure that the
381 // encoder queue is not blocked before we start sending it frames.
382 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100383 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200384 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800385 ASSERT_TRUE(event.Wait(5000));
386 }
387
Henrik Boström91aa7322020-04-28 12:24:33 +0200388 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200389 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200390 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200391 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200392 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200393 event.Set();
394 });
395 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200396 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200397 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200398
Henrik Boström91aa7322020-04-28 12:24:33 +0200399 void TriggerCpuUnderuse() {
400 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200401 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200402 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200403 event.Set();
404 });
405 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200406 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200407 }
kthelgason876222f2016-11-29 01:44:11 -0800408
Henrik Boström91aa7322020-04-28 12:24:33 +0200409 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200418 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200419 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200421 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200422 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200423 event.Set();
424 });
425 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200426 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200427 }
428
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200429 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100430 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200431 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
432 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200433 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700434};
435
Noah Richards51db4212019-06-12 06:59:12 -0700436// Simulates simulcast behavior and makes highest stream resolutions divisible
437// by 4.
438class CroppingVideoStreamFactory
439 : public VideoEncoderConfig::VideoStreamFactoryInterface {
440 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200441 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700442
443 private:
444 std::vector<VideoStream> CreateEncoderStreams(
445 int width,
446 int height,
447 const VideoEncoderConfig& encoder_config) override {
448 std::vector<VideoStream> streams = test::CreateVideoStreams(
449 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700450 return streams;
451 }
Noah Richards51db4212019-06-12 06:59:12 -0700452};
453
sprangb1ca0732017-02-01 08:38:12 -0800454class AdaptingFrameForwarder : public test::FrameForwarder {
455 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200456 explicit AdaptingFrameForwarder(TimeController* time_controller)
457 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700458 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800459
460 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200461 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800462 adaptation_enabled_ = enabled;
463 }
464
asaperssonfab67072017-04-04 05:51:49 -0700465 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200466 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800467 return adaptation_enabled_;
468 }
469
Henrik Boström1124ed12021-02-25 10:30:39 +0100470 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
471 // the resolution or frame rate was different than it is currently. If
472 // something else is modified, such as encoder resolutions, but the resolution
473 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700474 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200475 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700476 return last_wants_;
477 }
478
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200479 absl::optional<int> last_sent_width() const { return last_width_; }
480 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800481
sprangb1ca0732017-02-01 08:38:12 -0800482 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200483 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
484 time_controller_->AdvanceTime(TimeDelta::Millis(0));
485
sprangb1ca0732017-02-01 08:38:12 -0800486 int cropped_width = 0;
487 int cropped_height = 0;
488 int out_width = 0;
489 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700490 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200491 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
492 << "w=" << video_frame.width()
493 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700494 if (adapter_.AdaptFrameResolution(
495 video_frame.width(), video_frame.height(),
496 video_frame.timestamp_us() * 1000, &cropped_width,
497 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100498 VideoFrame adapted_frame =
499 VideoFrame::Builder()
500 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
501 nullptr, out_width, out_height))
502 .set_timestamp_rtp(99)
503 .set_timestamp_ms(99)
504 .set_rotation(kVideoRotation_0)
505 .build();
sprangc5d62e22017-04-02 23:53:04 -0700506 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100507 if (video_frame.has_update_rect()) {
508 adapted_frame.set_update_rect(
509 video_frame.update_rect().ScaleWithFrame(
510 video_frame.width(), video_frame.height(), 0, 0,
511 video_frame.width(), video_frame.height(), out_width,
512 out_height));
513 }
sprangc5d62e22017-04-02 23:53:04 -0700514 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800515 last_width_.emplace(adapted_frame.width());
516 last_height_.emplace(adapted_frame.height());
517 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200518 last_width_ = absl::nullopt;
519 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700520 }
sprangb1ca0732017-02-01 08:38:12 -0800521 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200522 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800523 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800524 last_width_.emplace(video_frame.width());
525 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800526 }
527 }
528
529 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
530 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200531 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100532 rtc::VideoSinkWants prev_wants = sink_wants_locked();
533 bool did_adapt =
534 prev_wants.max_pixel_count != wants.max_pixel_count ||
535 prev_wants.target_pixel_count != wants.target_pixel_count ||
536 prev_wants.max_framerate_fps != wants.max_framerate_fps;
537 if (did_adapt) {
538 last_wants_ = prev_wants;
539 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100540 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200541 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800542 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200543
544 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800545 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200546 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
547 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200548 absl::optional<int> last_width_;
549 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800550};
sprangc5d62e22017-04-02 23:53:04 -0700551
Niels Möller213618e2018-07-24 09:29:58 +0200552// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700553class MockableSendStatisticsProxy : public SendStatisticsProxy {
554 public:
555 MockableSendStatisticsProxy(Clock* clock,
556 const VideoSendStream::Config& config,
557 VideoEncoderConfig::ContentType content_type)
558 : SendStatisticsProxy(clock, config, content_type) {}
559
560 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200561 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700562 if (mock_stats_)
563 return *mock_stats_;
564 return SendStatisticsProxy::GetStats();
565 }
566
Niels Möller213618e2018-07-24 09:29:58 +0200567 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200568 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200569 if (mock_stats_)
570 return mock_stats_->input_frame_rate;
571 return SendStatisticsProxy::GetInputFrameRate();
572 }
sprangc5d62e22017-04-02 23:53:04 -0700573 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200574 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700575 mock_stats_.emplace(stats);
576 }
577
578 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200579 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700580 mock_stats_.reset();
581 }
582
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200583 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
584 on_frame_dropped_ = std::move(callback);
585 }
586
sprangc5d62e22017-04-02 23:53:04 -0700587 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200588 void OnFrameDropped(DropReason reason) override {
589 SendStatisticsProxy::OnFrameDropped(reason);
590 if (on_frame_dropped_)
591 on_frame_dropped_(reason);
592 }
593
Markus Handella3765182020-07-08 13:13:32 +0200594 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200595 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200596 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700597};
598
philipel9b058032020-02-10 11:30:00 +0100599class MockEncoderSelector
600 : public VideoEncoderFactory::EncoderSelectorInterface {
601 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200602 MOCK_METHOD(void,
603 OnCurrentEncoder,
604 (const SdpVideoFormat& format),
605 (override));
606 MOCK_METHOD(absl::optional<SdpVideoFormat>,
607 OnAvailableBitrate,
608 (const DataRate& rate),
609 (override));
610 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100611};
612
perkj803d97f2016-11-01 11:45:46 -0700613} // namespace
614
mflodmancc3d4422017-08-03 08:27:51 -0700615class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700616 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200617 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700618
mflodmancc3d4422017-08-03 08:27:51 -0700619 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700620 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700621 codec_width_(320),
622 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200623 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200624 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200625 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700626 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200627 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700628 video_send_config_,
629 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200630 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700631
632 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700633 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700634 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200635 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800636 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200637 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200638 video_send_config_.rtp.payload_name = "FAKE";
639 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700640
Per512ecb32016-09-23 15:52:06 +0200641 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200642 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200643 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
644 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
645 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100646 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700647
Niels Möllerf1338562018-04-26 09:51:47 +0200648 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800649 }
650
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100651 void ConfigureEncoder(
652 VideoEncoderConfig video_encoder_config,
653 VideoStreamEncoder::BitrateAllocationCallbackType
654 allocation_callback_type =
655 VideoStreamEncoder::BitrateAllocationCallbackType::
656 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700657 if (video_stream_encoder_)
658 video_stream_encoder_->Stop();
659 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200660 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100661 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700662 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
663 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700664 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700665 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
666 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200667 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800669 }
670
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100671 void ResetEncoder(const std::string& payload_name,
672 size_t num_streams,
673 size_t num_temporal_layers,
674 unsigned char num_spatial_layers,
675 bool screenshare,
676 VideoStreamEncoder::BitrateAllocationCallbackType
677 allocation_callback_type =
678 VideoStreamEncoder::BitrateAllocationCallbackType::
679 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200680 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800681
682 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200683 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
684 num_streams, &video_encoder_config);
685 for (auto& layer : video_encoder_config.simulcast_layers) {
686 layer.num_temporal_layers = num_temporal_layers;
687 layer.max_framerate = kDefaultFramerate;
688 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100689 video_encoder_config.max_bitrate_bps =
690 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700691 video_encoder_config.content_type =
692 screenshare ? VideoEncoderConfig::ContentType::kScreen
693 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700694 if (payload_name == "VP9") {
695 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
696 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200697 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700698 video_encoder_config.encoder_specific_settings =
699 new rtc::RefCountedObject<
700 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
701 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100702 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700703 }
704
sprang57c2fff2017-01-16 06:24:02 -0800705 VideoFrame CreateFrame(int64_t ntp_time_ms,
706 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100707 VideoFrame frame =
708 VideoFrame::Builder()
709 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
710 destruction_event, codec_width_, codec_height_))
711 .set_timestamp_rtp(99)
712 .set_timestamp_ms(99)
713 .set_rotation(kVideoRotation_0)
714 .build();
sprang57c2fff2017-01-16 06:24:02 -0800715 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700716 return frame;
717 }
718
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100719 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
720 rtc::Event* destruction_event,
721 int offset_x) const {
722 VideoFrame frame =
723 VideoFrame::Builder()
724 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
725 destruction_event, codec_width_, codec_height_))
726 .set_timestamp_rtp(99)
727 .set_timestamp_ms(99)
728 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100729 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100730 .build();
731 frame.set_ntp_time_ms(ntp_time_ms);
732 return frame;
733 }
734
sprang57c2fff2017-01-16 06:24:02 -0800735 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100736 VideoFrame frame =
737 VideoFrame::Builder()
738 .set_video_frame_buffer(
739 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
740 .set_timestamp_rtp(99)
741 .set_timestamp_ms(99)
742 .set_rotation(kVideoRotation_0)
743 .build();
sprang57c2fff2017-01-16 06:24:02 -0800744 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700745 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700746 return frame;
747 }
748
Evan Shrubsole895556e2020-10-05 09:15:13 +0200749 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
750 VideoFrame frame =
751 VideoFrame::Builder()
752 .set_video_frame_buffer(NV12Buffer::Create(width, height))
753 .set_timestamp_rtp(99)
754 .set_timestamp_ms(99)
755 .set_rotation(kVideoRotation_0)
756 .build();
757 frame.set_ntp_time_ms(ntp_time_ms);
758 frame.set_timestamp_us(ntp_time_ms * 1000);
759 return frame;
760 }
761
Noah Richards51db4212019-06-12 06:59:12 -0700762 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
763 rtc::Event* destruction_event,
764 int width,
765 int height) const {
766 VideoFrame frame =
767 VideoFrame::Builder()
768 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
769 destruction_event, width, height))
770 .set_timestamp_rtp(99)
771 .set_timestamp_ms(99)
772 .set_rotation(kVideoRotation_0)
773 .build();
774 frame.set_ntp_time_ms(ntp_time_ms);
775 return frame;
776 }
777
Evan Shrubsole895556e2020-10-05 09:15:13 +0200778 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
779 rtc::Event* destruction_event,
780 int width,
781 int height) const {
782 VideoFrame frame = VideoFrame::Builder()
783 .set_video_frame_buffer(
784 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
785 destruction_event, width, height))
786 .set_timestamp_rtp(99)
787 .set_timestamp_ms(99)
788 .set_rotation(kVideoRotation_0)
789 .build();
790 frame.set_ntp_time_ms(ntp_time_ms);
791 return frame;
792 }
793
Noah Richards51db4212019-06-12 06:59:12 -0700794 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
795 rtc::Event* destruction_event) const {
796 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
797 codec_height_);
798 }
799
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100800 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100802 DataRate::BitsPerSec(kTargetBitrateBps),
803 DataRate::BitsPerSec(kTargetBitrateBps),
804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100805
806 video_source_.IncomingCapturedFrame(
807 CreateFrame(1, codec_width_, codec_height_));
808 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200809 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100810 }
811
sprang4847ae62017-06-27 07:06:52 -0700812 void WaitForEncodedFrame(int64_t expected_ntp_time) {
813 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200814 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700815 }
816
817 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
818 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200819 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700820 return ok;
821 }
822
823 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
824 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200825 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700826 }
827
828 void ExpectDroppedFrame() {
829 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200830 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700831 }
832
833 bool WaitForFrame(int64_t timeout_ms) {
834 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700836 return ok;
837 }
838
perkj26091b12016-09-01 01:17:40 -0700839 class TestEncoder : public test::FakeEncoder {
840 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200841 explicit TestEncoder(TimeController* time_controller)
842 : FakeEncoder(time_controller->GetClock()),
843 time_controller_(time_controller) {
844 RTC_DCHECK(time_controller_);
845 }
perkj26091b12016-09-01 01:17:40 -0700846
asaperssonfab67072017-04-04 05:51:49 -0700847 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200848 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700849 return config_;
850 }
851
852 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200853 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700854 block_next_encode_ = true;
855 }
856
Erik Språngaed30702018-11-05 12:57:17 +0100857 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200858 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200859 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100860 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100861 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100862 info.scaling_settings = VideoEncoder::ScalingSettings(
863 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100864 }
865 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100866 for (int i = 0; i < kMaxSpatialLayers; ++i) {
867 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100868 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100869 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100870 for (int tid = 0; tid < num_layers; ++tid)
871 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100872 }
873 }
Erik Språngaed30702018-11-05 12:57:17 +0100874 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200875
876 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100877 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200878 info.apply_alignment_to_all_simulcast_layers =
879 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200880 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100881 return info;
kthelgason876222f2016-11-29 01:44:11 -0800882 }
883
Erik Språngb7cb7b52019-02-26 15:52:33 +0100884 int32_t RegisterEncodeCompleteCallback(
885 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200886 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100887 encoded_image_callback_ = callback;
888 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
889 }
890
perkjfa10b552016-10-02 23:45:26 -0700891 void ContinueEncode() { continue_encode_event_.Set(); }
892
893 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
894 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200895 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700896 EXPECT_EQ(timestamp_, timestamp);
897 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
898 }
899
kthelgason2fc52542017-03-03 00:24:41 -0800900 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200901 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800902 quality_scaling_ = b;
903 }
kthelgasonad9010c2017-02-14 00:46:51 -0800904
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100905 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200906 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100907 requested_resolution_alignment_ = requested_resolution_alignment;
908 }
909
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200910 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
911 MutexLock lock(&local_mutex_);
912 apply_alignment_to_all_simulcast_layers_ = b;
913 }
914
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100915 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100917 is_hardware_accelerated_ = is_hardware_accelerated;
918 }
919
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100920 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
921 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200922 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100923 temporal_layers_supported_[spatial_idx] = supported;
924 }
925
Sergey Silkin6456e352019-07-08 17:56:40 +0200926 void SetResolutionBitrateLimits(
927 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200928 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200929 resolution_bitrate_limits_ = thresholds;
930 }
931
sprangfe627f32017-03-29 08:24:59 -0700932 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200933 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700934 force_init_encode_failed_ = force_failure;
935 }
936
Niels Möller6bb5ab92019-01-11 11:11:10 +0100937 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200938 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100939 rate_factor_ = rate_factor;
940 }
941
Erik Språngd7329ca2019-02-21 21:19:53 +0100942 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200943 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100944 return last_framerate_;
945 }
946
Erik Språngd7329ca2019-02-21 21:19:53 +0100947 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200948 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100949 return last_update_rect_;
950 }
951
Niels Möller87e2d782019-03-07 10:18:23 +0100952 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200953 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100954 return last_frame_types_;
955 }
956
957 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100958 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100959 keyframe ? VideoFrameType::kVideoFrameKey
960 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100961 {
Markus Handella3765182020-07-08 13:13:32 +0200962 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100963 last_frame_types_ = frame_type;
964 }
Niels Möllerb859b322019-03-07 12:40:01 +0100965 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100966 }
967
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100968 void InjectEncodedImage(const EncodedImage& image,
969 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200970 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100971 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100972 }
973
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200974 void SetEncodedImageData(
975 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200976 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200977 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200978 }
979
Erik Språngd7329ca2019-02-21 21:19:53 +0100980 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200981 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100982 expect_null_frame_ = true;
983 }
984
Erik Språng5056af02019-09-02 15:53:11 +0200985 absl::optional<VideoEncoder::RateControlParameters>
986 GetAndResetLastRateControlSettings() {
987 auto settings = last_rate_control_settings_;
988 last_rate_control_settings_.reset();
989 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100990 }
991
Evan Shrubsole895556e2020-10-05 09:15:13 +0200992 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
993 MutexLock lock(&local_mutex_);
994 return last_input_pixel_format_;
995 }
996
Sergey Silkin5ee69672019-07-02 14:18:34 +0200997 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200998 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200999 return num_encoder_initializations_;
1000 }
1001
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001002 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001003 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001004 return num_set_rates_;
1005 }
1006
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001007 VideoCodec video_codec() const {
1008 MutexLock lock(&local_mutex_);
1009 return video_codec_;
1010 }
1011
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001012 void SetPreferredPixelFormats(
1013 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1014 pixel_formats) {
1015 MutexLock lock(&local_mutex_);
1016 preferred_pixel_formats_ = std::move(pixel_formats);
1017 }
1018
perkjfa10b552016-10-02 23:45:26 -07001019 private:
perkj26091b12016-09-01 01:17:40 -07001020 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001021 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001022 bool block_encode;
1023 {
Markus Handella3765182020-07-08 13:13:32 +02001024 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001025 if (expect_null_frame_) {
1026 EXPECT_EQ(input_image.timestamp(), 0u);
1027 EXPECT_EQ(input_image.width(), 1);
1028 last_frame_types_ = *frame_types;
1029 expect_null_frame_ = false;
1030 } else {
1031 EXPECT_GT(input_image.timestamp(), timestamp_);
1032 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1033 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1034 }
perkj26091b12016-09-01 01:17:40 -07001035
1036 timestamp_ = input_image.timestamp();
1037 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001038 last_input_width_ = input_image.width();
1039 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001040 block_encode = block_next_encode_;
1041 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001042 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001043 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001044 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001045 }
Niels Möllerb859b322019-03-07 12:40:01 +01001046 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001047 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001048 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001049
perkj26091b12016-09-01 01:17:40 -07001050 return result;
1051 }
1052
Niels Möller08ae7ce2020-09-23 15:58:12 +02001053 CodecSpecificInfo EncodeHook(
1054 EncodedImage& encoded_image,
1055 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001056 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001057 {
1058 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001059 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001060 }
1061 MutexLock lock(&local_mutex_);
1062 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001063 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001064 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001065 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001066 }
1067
sprangfe627f32017-03-29 08:24:59 -07001068 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001069 const Settings& settings) override {
1070 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001071
Markus Handella3765182020-07-08 13:13:32 +02001072 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001073 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001074
1075 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001076 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001077
Erik Språng82fad3d2018-03-21 09:57:23 +01001078 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001079 // Simulate setting up temporal layers, in order to validate the life
1080 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001081 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001082 frame_buffer_controller_ =
1083 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001084 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001085 if (force_init_encode_failed_) {
1086 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001087 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001088 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001089
Erik Språngb7cb7b52019-02-26 15:52:33 +01001090 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001091 return res;
1092 }
1093
Erik Språngb7cb7b52019-02-26 15:52:33 +01001094 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001095 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001096 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1097 initialized_ = EncoderState::kUninitialized;
1098 return FakeEncoder::Release();
1099 }
1100
Erik Språng16cb8f52019-04-12 13:59:09 +02001101 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001102 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001103 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001104 VideoBitrateAllocation adjusted_rate_allocation;
1105 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1106 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001107 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001108 adjusted_rate_allocation.SetBitrate(
1109 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001110 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001111 rate_factor_));
1112 }
1113 }
1114 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001115 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001116 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001117 RateControlParameters adjusted_paramters = parameters;
1118 adjusted_paramters.bitrate = adjusted_rate_allocation;
1119 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001120 }
1121
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001122 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001123 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001124 enum class EncoderState {
1125 kUninitialized,
1126 kInitializationFailed,
1127 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001128 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1129 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001130 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001131 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1132 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1133 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1134 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1135 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1136 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001137 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1138 false;
Markus Handella3765182020-07-08 13:13:32 +02001139 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001140 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1141 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001142 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001143 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001144 absl::optional<bool>
1145 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001146 local_mutex_);
1147 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1148 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1149 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001150 absl::optional<VideoEncoder::RateControlParameters>
1151 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001152 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1153 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001154 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001155 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001156 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1157 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001158 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001159 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001160 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001161 RTC_GUARDED_BY(local_mutex_);
1162 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001163 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001164 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1165 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001166 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1167 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001168 };
1169
mflodmancc3d4422017-08-03 08:27:51 -07001170 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001171 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001172 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1173 : time_controller_(time_controller), test_encoder_(test_encoder) {
1174 RTC_DCHECK(time_controller_);
1175 }
perkj26091b12016-09-01 01:17:40 -07001176
perkj26091b12016-09-01 01:17:40 -07001177 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001178 EXPECT_TRUE(
1179 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1180 }
1181
1182 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1183 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001184 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001185 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001186 return false;
perkj26091b12016-09-01 01:17:40 -07001187 {
Markus Handella3765182020-07-08 13:13:32 +02001188 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001189 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001190 }
1191 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001192 return true;
perkj26091b12016-09-01 01:17:40 -07001193 }
1194
sprangb1ca0732017-02-01 08:38:12 -08001195 void WaitForEncodedFrame(uint32_t expected_width,
1196 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001197 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001198 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001199 }
1200
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001201 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001202 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001203 uint32_t width = 0;
1204 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001205 {
Markus Handella3765182020-07-08 13:13:32 +02001206 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001207 width = last_width_;
1208 height = last_height_;
1209 }
1210 EXPECT_EQ(expected_height, height);
1211 EXPECT_EQ(expected_width, width);
1212 }
1213
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001214 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1215 VideoRotation rotation;
1216 {
Markus Handella3765182020-07-08 13:13:32 +02001217 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001218 rotation = last_rotation_;
1219 }
1220 EXPECT_EQ(expected_rotation, rotation);
1221 }
1222
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001223 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001224
sprangc5d62e22017-04-02 23:53:04 -07001225 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001226 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1227 bool ret = encoded_frame_event_.Wait(timeout_ms);
1228 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1229 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001230 }
1231
perkj26091b12016-09-01 01:17:40 -07001232 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001233 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001234 expect_frames_ = false;
1235 }
1236
asaperssonfab67072017-04-04 05:51:49 -07001237 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001238 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001239 return number_of_reconfigurations_;
1240 }
1241
asaperssonfab67072017-04-04 05:51:49 -07001242 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001243 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001244 return min_transmit_bitrate_bps_;
1245 }
1246
Erik Språngd7329ca2019-02-21 21:19:53 +01001247 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001248 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001249 num_expected_layers_ = num_layers;
1250 }
1251
Erik Språngb7cb7b52019-02-26 15:52:33 +01001252 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001253 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001254 return last_capture_time_ms_;
1255 }
1256
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001257 const EncodedImage& GetLastEncodedImage() {
1258 MutexLock lock(&mutex_);
1259 return last_encoded_image_;
1260 }
1261
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001262 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001263 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001264 return std::move(last_encoded_image_data_);
1265 }
1266
Per Kjellanderdcef6412020-10-07 15:09:05 +02001267 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1268 MutexLock lock(&mutex_);
1269 return last_bitrate_allocation_;
1270 }
1271
1272 int number_of_bitrate_allocations() const {
1273 MutexLock lock(&mutex_);
1274 return number_of_bitrate_allocations_;
1275 }
1276
Per Kjellandera9434842020-10-15 17:53:22 +02001277 VideoLayersAllocation GetLastVideoLayersAllocation() {
1278 MutexLock lock(&mutex_);
1279 return last_layers_allocation_;
1280 }
1281
1282 int number_of_layers_allocations() const {
1283 MutexLock lock(&mutex_);
1284 return number_of_layers_allocations_;
1285 }
1286
perkj26091b12016-09-01 01:17:40 -07001287 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001288 Result OnEncodedImage(
1289 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001290 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001291 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001292 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001293 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001294 last_encoded_image_data_ = std::vector<uint8_t>(
1295 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001296 uint32_t timestamp = encoded_image.Timestamp();
1297 if (last_timestamp_ != timestamp) {
1298 num_received_layers_ = 1;
1299 } else {
1300 ++num_received_layers_;
1301 }
1302 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001303 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001304 last_width_ = encoded_image._encodedWidth;
1305 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001306 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001307 if (num_received_layers_ == num_expected_layers_) {
1308 encoded_frame_event_.Set();
1309 }
sprangb1ca0732017-02-01 08:38:12 -08001310 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001311 }
1312
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001313 void OnEncoderConfigurationChanged(
1314 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001315 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001316 VideoEncoderConfig::ContentType content_type,
1317 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001318 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001319 ++number_of_reconfigurations_;
1320 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1321 }
1322
Per Kjellanderdcef6412020-10-07 15:09:05 +02001323 void OnBitrateAllocationUpdated(
1324 const VideoBitrateAllocation& allocation) override {
1325 MutexLock lock(&mutex_);
1326 ++number_of_bitrate_allocations_;
1327 last_bitrate_allocation_ = allocation;
1328 }
1329
Per Kjellandera9434842020-10-15 17:53:22 +02001330 void OnVideoLayersAllocationUpdated(
1331 VideoLayersAllocation allocation) override {
1332 MutexLock lock(&mutex_);
1333 ++number_of_layers_allocations_;
1334 last_layers_allocation_ = allocation;
1335 rtc::StringBuilder log;
1336 for (const auto& layer : allocation.active_spatial_layers) {
1337 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1338 << "[";
1339 for (const auto target_bitrate :
1340 layer.target_bitrate_per_temporal_layer) {
1341 log << target_bitrate.kbps() << ",";
1342 }
1343 log << "]";
1344 }
1345 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1346 }
1347
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001348 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001349 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001350 TestEncoder* test_encoder_;
1351 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001352 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001353 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001354 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001355 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001356 uint32_t last_height_ = 0;
1357 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001358 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001359 size_t num_expected_layers_ = 1;
1360 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001361 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001362 int number_of_reconfigurations_ = 0;
1363 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001364 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1365 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001366 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1367 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001368 };
1369
Sergey Silkin5ee69672019-07-02 14:18:34 +02001370 class VideoBitrateAllocatorProxyFactory
1371 : public VideoBitrateAllocatorFactory {
1372 public:
1373 VideoBitrateAllocatorProxyFactory()
1374 : bitrate_allocator_factory_(
1375 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1376
1377 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1378 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001379 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001380 codec_config_ = codec;
1381 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1382 }
1383
1384 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001385 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001386 return codec_config_;
1387 }
1388
1389 private:
1390 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1391
Markus Handella3765182020-07-08 13:13:32 +02001392 mutable Mutex mutex_;
1393 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001394 };
1395
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001396 Clock* clock() { return time_controller_.GetClock(); }
1397 void AdvanceTime(TimeDelta duration) {
1398 time_controller_.AdvanceTime(duration);
1399 }
1400
1401 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1402
1403 protected:
1404 virtual TaskQueueFactory* GetTaskQueueFactory() {
1405 return time_controller_.GetTaskQueueFactory();
1406 }
1407
1408 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001409 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001410 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001411 int codec_width_;
1412 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001413 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001414 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001415 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001416 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001417 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001418 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001419 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001420 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001421};
1422
mflodmancc3d4422017-08-03 08:27:51 -07001423TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001425 DataRate::BitsPerSec(kTargetBitrateBps),
1426 DataRate::BitsPerSec(kTargetBitrateBps),
1427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001428 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001429 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001430 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001431 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001432 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001433}
1434
mflodmancc3d4422017-08-03 08:27:51 -07001435TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001436 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001437 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001438 // The encoder will cache up to one frame for a short duration. Adding two
1439 // frames means that the first frame will be dropped and the second frame will
1440 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001441 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001442 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001443 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001444 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001445
Henrik Boström381d1092020-05-12 18:49:07 +02001446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001447 DataRate::BitsPerSec(kTargetBitrateBps),
1448 DataRate::BitsPerSec(kTargetBitrateBps),
1449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001450
Sebastian Janssona3177052018-04-10 13:05:49 +02001451 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001452 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001453 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1454
1455 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001456 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001457}
1458
mflodmancc3d4422017-08-03 08:27:51 -07001459TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001460 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001461 DataRate::BitsPerSec(kTargetBitrateBps),
1462 DataRate::BitsPerSec(kTargetBitrateBps),
1463 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001464 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001465 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001466
Henrik Boström381d1092020-05-12 18:49:07 +02001467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1468 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1469 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001470 // The encoder will cache up to one frame for a short duration. Adding two
1471 // frames means that the first frame will be dropped and the second frame will
1472 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001473 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001474 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001475
Henrik Boström381d1092020-05-12 18:49:07 +02001476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001477 DataRate::BitsPerSec(kTargetBitrateBps),
1478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001480 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001481 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1482 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001483 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001484}
1485
mflodmancc3d4422017-08-03 08:27:51 -07001486TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001488 DataRate::BitsPerSec(kTargetBitrateBps),
1489 DataRate::BitsPerSec(kTargetBitrateBps),
1490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001491 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001492 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001493
1494 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001495 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001496
perkja49cbd32016-09-16 07:53:41 -07001497 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001498 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001499 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001500}
1501
mflodmancc3d4422017-08-03 08:27:51 -07001502TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001504 DataRate::BitsPerSec(kTargetBitrateBps),
1505 DataRate::BitsPerSec(kTargetBitrateBps),
1506 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001507
perkja49cbd32016-09-16 07:53:41 -07001508 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001509 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001510
mflodmancc3d4422017-08-03 08:27:51 -07001511 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001512 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001513 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001514 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1515 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001516}
1517
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001518class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1519 public:
1520 VideoStreamEncoderBlockedTest() {}
1521
1522 TaskQueueFactory* GetTaskQueueFactory() override {
1523 return task_queue_factory_.get();
1524 }
1525
1526 private:
1527 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1528 CreateDefaultTaskQueueFactory();
1529};
1530
1531TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001532 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001533 DataRate::BitsPerSec(kTargetBitrateBps),
1534 DataRate::BitsPerSec(kTargetBitrateBps),
1535 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001536
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001537 int dropped_count = 0;
1538 stats_proxy_->SetDroppedFrameCallback(
1539 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1540 ++dropped_count;
1541 });
1542
perkj26091b12016-09-01 01:17:40 -07001543 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001544 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001545 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001546 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1547 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001548 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1549 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001550 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001551 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001552
mflodmancc3d4422017-08-03 08:27:51 -07001553 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001554
1555 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001556}
1557
Noah Richards51db4212019-06-12 06:59:12 -07001558TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001560 DataRate::BitsPerSec(kTargetBitrateBps),
1561 DataRate::BitsPerSec(kTargetBitrateBps),
1562 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001563
1564 rtc::Event frame_destroyed_event;
1565 video_source_.IncomingCapturedFrame(
1566 CreateFakeNativeFrame(1, &frame_destroyed_event));
1567 ExpectDroppedFrame();
1568 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1569 video_stream_encoder_->Stop();
1570}
1571
1572TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1573 // Use the cropping factory.
1574 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001575 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001576 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1577 kMaxPayloadLength);
1578 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1579
1580 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001582 DataRate::BitsPerSec(kTargetBitrateBps),
1583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001585 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1586 WaitForEncodedFrame(1);
1587 // The encoder will have been configured once.
1588 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1589 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1590 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1591
1592 // Now send in a fake frame that needs to be cropped as the width/height
1593 // aren't divisible by 4 (see CreateEncoderStreams above).
1594 rtc::Event frame_destroyed_event;
1595 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1596 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1597 ExpectDroppedFrame();
1598 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1599 video_stream_encoder_->Stop();
1600}
1601
Evan Shrubsole895556e2020-10-05 09:15:13 +02001602TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1604 DataRate::BitsPerSec(kTargetBitrateBps),
1605 DataRate::BitsPerSec(kTargetBitrateBps),
1606 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1607
1608 video_source_.IncomingCapturedFrame(
1609 CreateNV12Frame(1, codec_width_, codec_height_));
1610 WaitForEncodedFrame(1);
1611 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1612 fake_encoder_.GetLastInputPixelFormat());
1613 video_stream_encoder_->Stop();
1614}
1615
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001616TEST_F(VideoStreamEncoderTest,
1617 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1619 DataRate::BitsPerSec(kTargetBitrateBps),
1620 DataRate::BitsPerSec(kTargetBitrateBps),
1621 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1622
1623 fake_encoder_.SetPreferredPixelFormats({});
1624
1625 rtc::Event frame_destroyed_event;
1626 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1627 1, &frame_destroyed_event, codec_width_, codec_height_));
1628 WaitForEncodedFrame(1);
1629 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1630 fake_encoder_.GetLastInputPixelFormat());
1631 video_stream_encoder_->Stop();
1632}
1633
1634TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1636 DataRate::BitsPerSec(kTargetBitrateBps),
1637 DataRate::BitsPerSec(kTargetBitrateBps),
1638 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1639
1640 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1641
1642 rtc::Event frame_destroyed_event;
1643 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1644 1, &frame_destroyed_event, codec_width_, codec_height_));
1645 WaitForEncodedFrame(1);
1646 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1647 fake_encoder_.GetLastInputPixelFormat());
1648 video_stream_encoder_->Stop();
1649}
1650
1651TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1653 DataRate::BitsPerSec(kTargetBitrateBps),
1654 DataRate::BitsPerSec(kTargetBitrateBps),
1655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1656
1657 // Fake NV12 native frame does not allow mapping to I444.
1658 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1659
1660 rtc::Event frame_destroyed_event;
1661 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1662 1, &frame_destroyed_event, codec_width_, codec_height_));
1663 WaitForEncodedFrame(1);
1664 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1665 fake_encoder_.GetLastInputPixelFormat());
1666 video_stream_encoder_->Stop();
1667}
1668
Evan Shrubsole895556e2020-10-05 09:15:13 +02001669TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1671 DataRate::BitsPerSec(kTargetBitrateBps),
1672 DataRate::BitsPerSec(kTargetBitrateBps),
1673 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1674
1675 rtc::Event frame_destroyed_event;
1676 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1677 1, &frame_destroyed_event, codec_width_, codec_height_));
1678 WaitForEncodedFrame(1);
1679 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1680 fake_encoder_.GetLastInputPixelFormat());
1681 video_stream_encoder_->Stop();
1682}
1683
Ying Wang9b881ab2020-02-07 14:29:32 +01001684TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001686 DataRate::BitsPerSec(kTargetBitrateBps),
1687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001689 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1690 WaitForEncodedFrame(1);
1691
Henrik Boström381d1092020-05-12 18:49:07 +02001692 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001693 DataRate::BitsPerSec(kTargetBitrateBps),
1694 DataRate::BitsPerSec(kTargetBitrateBps),
1695 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001696 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1697 // frames. Adding two frames means that the first frame will be dropped and
1698 // the second frame will be sent to the encoder.
1699 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1700 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1701 WaitForEncodedFrame(3);
1702 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1703 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1704 WaitForEncodedFrame(5);
1705 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1706 video_stream_encoder_->Stop();
1707}
1708
mflodmancc3d4422017-08-03 08:27:51 -07001709TEST_F(VideoStreamEncoderTest,
1710 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001712 DataRate::BitsPerSec(kTargetBitrateBps),
1713 DataRate::BitsPerSec(kTargetBitrateBps),
1714 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001715 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001716
1717 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001718 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001719 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001720 // The encoder will have been configured once when the first frame is
1721 // received.
1722 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001723
1724 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001725 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001726 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001727 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001728 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001729
1730 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001731 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001732 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001733 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001734 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001735
mflodmancc3d4422017-08-03 08:27:51 -07001736 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001737}
1738
mflodmancc3d4422017-08-03 08:27:51 -07001739TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001741 DataRate::BitsPerSec(kTargetBitrateBps),
1742 DataRate::BitsPerSec(kTargetBitrateBps),
1743 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001744
1745 // Capture a frame and wait for it to synchronize with the encoder thread.
1746 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001747 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001748 // The encoder will have been configured once.
1749 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001750 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1751 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1752
1753 codec_width_ *= 2;
1754 codec_height_ *= 2;
1755 // Capture a frame with a higher resolution and wait for it to synchronize
1756 // with the encoder thread.
1757 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001758 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001759 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1760 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001761 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001762
mflodmancc3d4422017-08-03 08:27:51 -07001763 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001764}
1765
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001766TEST_F(VideoStreamEncoderTest,
1767 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps),
1771 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001772
1773 // Capture a frame and wait for it to synchronize with the encoder thread.
1774 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1775 WaitForEncodedFrame(1);
1776
1777 VideoEncoderConfig video_encoder_config;
1778 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1779 // Changing the max payload data length recreates encoder.
1780 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1781 kMaxPayloadLength / 2);
1782
1783 // Capture a frame and wait for it to synchronize with the encoder thread.
1784 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1785 WaitForEncodedFrame(2);
1786 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1787
1788 video_stream_encoder_->Stop();
1789}
1790
Sergey Silkin5ee69672019-07-02 14:18:34 +02001791TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001793 DataRate::BitsPerSec(kTargetBitrateBps),
1794 DataRate::BitsPerSec(kTargetBitrateBps),
1795 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001796
1797 VideoEncoderConfig video_encoder_config;
1798 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1799 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1800 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1801 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1802 kMaxPayloadLength);
1803
1804 // Capture a frame and wait for it to synchronize with the encoder thread.
1805 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1806 WaitForEncodedFrame(1);
1807 // The encoder will have been configured once when the first frame is
1808 // received.
1809 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1810 EXPECT_EQ(kTargetBitrateBps,
1811 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1812 EXPECT_EQ(kStartBitrateBps,
1813 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1814
Sergey Silkin6456e352019-07-08 17:56:40 +02001815 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1816 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001817 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1818 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1819 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1820 kMaxPayloadLength);
1821
1822 // Capture a frame and wait for it to synchronize with the encoder thread.
1823 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1824 WaitForEncodedFrame(2);
1825 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1826 // Bitrate limits have changed - rate allocator should be reconfigured,
1827 // encoder should not be reconfigured.
1828 EXPECT_EQ(kTargetBitrateBps * 2,
1829 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1830 EXPECT_EQ(kStartBitrateBps * 2,
1831 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1832 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1833
1834 video_stream_encoder_->Stop();
1835}
1836
Sergey Silkin6456e352019-07-08 17:56:40 +02001837TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001838 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001840 DataRate::BitsPerSec(kTargetBitrateBps),
1841 DataRate::BitsPerSec(kTargetBitrateBps),
1842 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001843
Sergey Silkincd02eba2020-01-20 14:48:40 +01001844 const uint32_t kMinEncBitrateKbps = 100;
1845 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001846 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001847 /*frame_size_pixels=*/codec_width_ * codec_height_,
1848 /*min_start_bitrate_bps=*/0,
1849 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1850 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001851 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1852
Sergey Silkincd02eba2020-01-20 14:48:40 +01001853 VideoEncoderConfig video_encoder_config;
1854 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1855 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1856 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1857 (kMinEncBitrateKbps + 1) * 1000;
1858 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1859 kMaxPayloadLength);
1860
1861 // When both encoder and app provide bitrate limits, the intersection of
1862 // provided sets should be used.
1863 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1864 WaitForEncodedFrame(1);
1865 EXPECT_EQ(kMaxEncBitrateKbps,
1866 bitrate_allocator_factory_.codec_config().maxBitrate);
1867 EXPECT_EQ(kMinEncBitrateKbps + 1,
1868 bitrate_allocator_factory_.codec_config().minBitrate);
1869
1870 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1871 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1872 (kMinEncBitrateKbps - 1) * 1000;
1873 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1874 kMaxPayloadLength);
1875 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001876 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001877 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001878 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001879 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001880 bitrate_allocator_factory_.codec_config().minBitrate);
1881
Sergey Silkincd02eba2020-01-20 14:48:40 +01001882 video_stream_encoder_->Stop();
1883}
1884
1885TEST_F(VideoStreamEncoderTest,
1886 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001888 DataRate::BitsPerSec(kTargetBitrateBps),
1889 DataRate::BitsPerSec(kTargetBitrateBps),
1890 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001891
1892 const uint32_t kMinAppBitrateKbps = 100;
1893 const uint32_t kMaxAppBitrateKbps = 200;
1894 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1895 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1896 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1897 /*frame_size_pixels=*/codec_width_ * codec_height_,
1898 /*min_start_bitrate_bps=*/0,
1899 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1900 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1901 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1902
1903 VideoEncoderConfig video_encoder_config;
1904 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1905 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1906 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1907 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001908 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1909 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001910
Sergey Silkincd02eba2020-01-20 14:48:40 +01001911 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1912 WaitForEncodedFrame(1);
1913 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001914 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001915 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001916 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001917
1918 video_stream_encoder_->Stop();
1919}
1920
1921TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001922 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001924 DataRate::BitsPerSec(kTargetBitrateBps),
1925 DataRate::BitsPerSec(kTargetBitrateBps),
1926 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001927
1928 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001929 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001930 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001931 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001932 fake_encoder_.SetResolutionBitrateLimits(
1933 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1934
1935 VideoEncoderConfig video_encoder_config;
1936 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1937 video_encoder_config.max_bitrate_bps = 0;
1938 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1939 kMaxPayloadLength);
1940
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001941 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001942 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1943 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001944 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1945 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001946 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1948
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001950 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1951 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001952 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1953 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001954 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1955 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1956
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001957 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001958 // encoder for 360p should be used.
1959 video_source_.IncomingCapturedFrame(
1960 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1961 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1963 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001964 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1965 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1966
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001967 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001968 // ignored.
1969 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1970 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001971 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1972 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001973 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1974 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001975 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1976 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001977 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1978 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1979
1980 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1981 // for 270p should be used.
1982 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1983 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001984 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1985 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001986 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1987 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1988
1989 video_stream_encoder_->Stop();
1990}
1991
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001992TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001994 DataRate::BitsPerSec(kTargetBitrateBps),
1995 DataRate::BitsPerSec(kTargetBitrateBps),
1996 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001997
1998 VideoEncoderConfig video_encoder_config;
1999 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2000 video_encoder_config.max_bitrate_bps = 0;
2001 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2002 kMaxPayloadLength);
2003
2004 // Encode 720p frame to get the default encoder target bitrate.
2005 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2006 WaitForEncodedFrame(1);
2007 const uint32_t kDefaultTargetBitrateFor720pKbps =
2008 bitrate_allocator_factory_.codec_config()
2009 .simulcastStream[0]
2010 .targetBitrate;
2011
2012 // Set the max recommended encoder bitrate to something lower than the default
2013 // target bitrate.
2014 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2015 1280 * 720, 10 * 1000, 10 * 1000,
2016 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2017 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2018
2019 // Change resolution to trigger encoder reinitialization.
2020 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2021 WaitForEncodedFrame(2);
2022 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2023 WaitForEncodedFrame(3);
2024
2025 // Ensure the target bitrate is capped by the max bitrate.
2026 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2027 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2028 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2029 .simulcastStream[0]
2030 .targetBitrate *
2031 1000,
2032 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2033
2034 video_stream_encoder_->Stop();
2035}
2036
Åsa Perssona7e34d32021-01-20 15:36:13 +01002037TEST_F(VideoStreamEncoderTest,
2038 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2039 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2040 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2041 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2042 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2043 fake_encoder_.SetResolutionBitrateLimits(
2044 {kEncoderLimits270p, kEncoderLimits360p});
2045
2046 // Two streams, highest stream active.
2047 VideoEncoderConfig config;
2048 const int kNumStreams = 2;
2049 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2050 config.max_bitrate_bps = 0;
2051 config.simulcast_layers[0].active = false;
2052 config.simulcast_layers[1].active = true;
2053 config.video_stream_factory =
2054 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2055 "VP8", /*max qp*/ 56, /*screencast*/ false,
2056 /*screenshare enabled*/ false);
2057 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2058
2059 // The encoder bitrate limits for 270p should be used.
2060 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2061 EXPECT_FALSE(WaitForFrame(1000));
2062 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2063 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2064 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2065 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2066 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2067
2068 // The encoder bitrate limits for 360p should be used.
2069 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2070 EXPECT_FALSE(WaitForFrame(1000));
2071 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2072 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2073 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2074 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2075
2076 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2077 video_source_.IncomingCapturedFrame(
2078 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2079 EXPECT_FALSE(WaitForFrame(1000));
2080 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2081 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2082 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2083 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2084
2085 // Resolution higher than 360p. Encoder limits should be ignored.
2086 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2087 EXPECT_FALSE(WaitForFrame(1000));
2088 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2089 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2090 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2091 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2092 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2093 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2094 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2095 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2096
2097 // Resolution lower than 270p. The encoder limits for 270p should be used.
2098 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2099 EXPECT_FALSE(WaitForFrame(1000));
2100 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2101 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2102 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2103 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2104
2105 video_stream_encoder_->Stop();
2106}
2107
2108TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002109 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2110 // Two streams, highest stream active.
2111 VideoEncoderConfig config;
2112 const int kNumStreams = 2;
2113 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2114 config.max_bitrate_bps = 0;
2115 config.simulcast_layers[0].active = false;
2116 config.simulcast_layers[1].active = true;
2117 config.video_stream_factory =
2118 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2119 "VP8", /*max qp*/ 56, /*screencast*/ false,
2120 /*screenshare enabled*/ false);
2121 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2122
2123 // Default bitrate limits for 270p should be used.
2124 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2125 kDefaultLimits270p =
2126 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002127 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002128 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2129 EXPECT_FALSE(WaitForFrame(1000));
2130 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2131 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2132 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2133 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2134 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2135
2136 // Default bitrate limits for 360p should be used.
2137 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2138 kDefaultLimits360p =
2139 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002140 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002141 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2142 EXPECT_FALSE(WaitForFrame(1000));
2143 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2144 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2145 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2146 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2147
2148 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2149 video_source_.IncomingCapturedFrame(
2150 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2151 EXPECT_FALSE(WaitForFrame(1000));
2152 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2153 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2154 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2155 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2156
2157 // Default bitrate limits for 540p should be used.
2158 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2159 kDefaultLimits540p =
2160 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002161 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002162 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2163 EXPECT_FALSE(WaitForFrame(1000));
2164 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2165 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2166 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2167 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2168
2169 video_stream_encoder_->Stop();
2170}
2171
2172TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002173 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2174 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2175 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2176 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2177 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2178 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2179 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2180 fake_encoder_.SetResolutionBitrateLimits(
2181 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2182
2183 // Three streams, middle stream active.
2184 VideoEncoderConfig config;
2185 const int kNumStreams = 3;
2186 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2187 config.simulcast_layers[0].active = false;
2188 config.simulcast_layers[1].active = true;
2189 config.simulcast_layers[2].active = false;
2190 config.video_stream_factory =
2191 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2192 "VP8", /*max qp*/ 56, /*screencast*/ false,
2193 /*screenshare enabled*/ false);
2194 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2195
2196 // The encoder bitrate limits for 360p should be used.
2197 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2198 EXPECT_FALSE(WaitForFrame(1000));
2199 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2200 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2201 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2202 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2203 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2204
2205 // The encoder bitrate limits for 270p should be used.
2206 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2207 EXPECT_FALSE(WaitForFrame(1000));
2208 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2209 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2211 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2212
2213 video_stream_encoder_->Stop();
2214}
2215
2216TEST_F(VideoStreamEncoderTest,
2217 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2218 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2219 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2220 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2221 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2222 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2223 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2224 fake_encoder_.SetResolutionBitrateLimits(
2225 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2226
2227 // Three streams, lowest stream active.
2228 VideoEncoderConfig config;
2229 const int kNumStreams = 3;
2230 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2231 config.simulcast_layers[0].active = true;
2232 config.simulcast_layers[1].active = false;
2233 config.simulcast_layers[2].active = false;
2234 config.video_stream_factory =
2235 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2236 "VP8", /*max qp*/ 56, /*screencast*/ false,
2237 /*screenshare enabled*/ false);
2238 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2239
2240 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2241 // on lowest stream, limits for 270p should not be used
2242 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2243 EXPECT_FALSE(WaitForFrame(1000));
2244 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2245 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2246 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2247 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2248 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2249
2250 video_stream_encoder_->Stop();
2251}
2252
2253TEST_F(VideoStreamEncoderTest,
2254 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2255 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2256 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2257 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2258 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2259 fake_encoder_.SetResolutionBitrateLimits(
2260 {kEncoderLimits270p, kEncoderLimits360p});
2261 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2262
2263 // Two streams, highest stream active.
2264 VideoEncoderConfig config;
2265 const int kNumStreams = 2;
2266 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2267 config.simulcast_layers[0].active = false;
2268 config.simulcast_layers[1].active = true;
2269 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2270 config.video_stream_factory =
2271 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2272 "VP8", /*max qp*/ 56, /*screencast*/ false,
2273 /*screenshare enabled*/ false);
2274 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2275
2276 // The encoder bitrate limits for 270p should be used.
2277 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2278 EXPECT_FALSE(WaitForFrame(1000));
2279 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2280 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2281 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2282 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2283 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2284
2285 // The max configured bitrate is less than the encoder limit for 360p.
2286 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2287 EXPECT_FALSE(WaitForFrame(1000));
2288 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2289 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2290 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2291 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2292
2293 video_stream_encoder_->Stop();
2294}
2295
mflodmancc3d4422017-08-03 08:27:51 -07002296TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002297 EXPECT_TRUE(video_source_.has_sinks());
2298 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002299 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002300 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002301 EXPECT_FALSE(video_source_.has_sinks());
2302 EXPECT_TRUE(new_video_source.has_sinks());
2303
mflodmancc3d4422017-08-03 08:27:51 -07002304 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002305}
2306
mflodmancc3d4422017-08-03 08:27:51 -07002307TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002308 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002309 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002310 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002311 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002312}
2313
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002314class ResolutionAlignmentTest
2315 : public VideoStreamEncoderTest,
2316 public ::testing::WithParamInterface<
2317 ::testing::tuple<int, std::vector<double>>> {
2318 public:
2319 ResolutionAlignmentTest()
2320 : requested_alignment_(::testing::get<0>(GetParam())),
2321 scale_factors_(::testing::get<1>(GetParam())) {}
2322
2323 protected:
2324 const int requested_alignment_;
2325 const std::vector<double> scale_factors_;
2326};
2327
2328INSTANTIATE_TEST_SUITE_P(
2329 AlignmentAndScaleFactors,
2330 ResolutionAlignmentTest,
2331 ::testing::Combine(
2332 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2333 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2334 std::vector<double>{-1.0, -1.0},
2335 std::vector<double>{-1.0, -1.0, -1.0},
2336 std::vector<double>{4.0, 2.0, 1.0},
2337 std::vector<double>{9999.0, -1.0, 1.0},
2338 std::vector<double>{3.99, 2.01, 1.0},
2339 std::vector<double>{4.9, 1.7, 1.25},
2340 std::vector<double>{10.0, 4.0, 3.0},
2341 std::vector<double>{1.75, 3.5},
2342 std::vector<double>{1.5, 2.5},
2343 std::vector<double>{1.3, 1.0})));
2344
2345TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2346 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002347 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002348 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2349 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2350
2351 // Fill config with the scaling factor by which to reduce encoding size.
2352 const int num_streams = scale_factors_.size();
2353 VideoEncoderConfig config;
2354 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2355 for (int i = 0; i < num_streams; ++i) {
2356 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2357 }
2358 config.video_stream_factory =
2359 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2360 "VP8", /*max qp*/ 56, /*screencast*/ false,
2361 /*screenshare enabled*/ false);
2362 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2363
Henrik Boström381d1092020-05-12 18:49:07 +02002364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002365 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2366 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2367 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2368 // Wait for all layers before triggering event.
2369 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002370
2371 // On the 1st frame, we should have initialized the encoder and
2372 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002373 int64_t timestamp_ms = kFrameIntervalMs;
2374 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2375 WaitForEncodedFrame(timestamp_ms);
2376 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002377
2378 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2379 // (It's up the to the encoder to potentially drop the previous frame,
2380 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002381 timestamp_ms += kFrameIntervalMs;
2382 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2383 WaitForEncodedFrame(timestamp_ms);
2384 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2385
2386 VideoCodec codec = fake_encoder_.video_codec();
2387 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2388 // Frame size should be a multiple of the requested alignment.
2389 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2390 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2391 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2392 // Aspect ratio should match.
2393 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2394 codec.height * codec.simulcastStream[i].width);
2395 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002396
2397 video_stream_encoder_->Stop();
2398}
2399
Jonathan Yubc771b72017-12-08 17:04:29 -08002400TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2401 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002402 const int kWidth = 1280;
2403 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002404
2405 // We rely on the automatic resolution adaptation, but we handle framerate
2406 // adaptation manually by mocking the stats proxy.
2407 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002408
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002409 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002411 DataRate::BitsPerSec(kTargetBitrateBps),
2412 DataRate::BitsPerSec(kTargetBitrateBps),
2413 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002414 video_stream_encoder_->SetSource(&video_source_,
2415 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002416 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002417 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002418 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002419 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2420
Jonathan Yubc771b72017-12-08 17:04:29 -08002421 // Adapt down as far as possible.
2422 rtc::VideoSinkWants last_wants;
2423 int64_t t = 1;
2424 int loop_count = 0;
2425 do {
2426 ++loop_count;
2427 last_wants = video_source_.sink_wants();
2428
2429 // Simulate the framerate we've been asked to adapt to.
2430 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2431 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2432 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2433 mock_stats.input_frame_rate = fps;
2434 stats_proxy_->SetMockStats(mock_stats);
2435
2436 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2437 sink_.WaitForEncodedFrame(t);
2438 t += frame_interval_ms;
2439
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002441 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002442 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002443 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2444 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002445 } while (video_source_.sink_wants().max_pixel_count <
2446 last_wants.max_pixel_count ||
2447 video_source_.sink_wants().max_framerate_fps <
2448 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002449
Jonathan Yubc771b72017-12-08 17:04:29 -08002450 // Verify that we've adapted all the way down.
2451 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002452 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002453 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2454 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002455 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002456 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2457 *video_source_.last_sent_height());
2458 EXPECT_EQ(kMinBalancedFramerateFps,
2459 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002460
Jonathan Yubc771b72017-12-08 17:04:29 -08002461 // Adapt back up the same number of times we adapted down.
2462 for (int i = 0; i < loop_count - 1; ++i) {
2463 last_wants = video_source_.sink_wants();
2464
2465 // Simulate the framerate we've been asked to adapt to.
2466 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2467 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2468 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2469 mock_stats.input_frame_rate = fps;
2470 stats_proxy_->SetMockStats(mock_stats);
2471
2472 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2473 sink_.WaitForEncodedFrame(t);
2474 t += frame_interval_ms;
2475
Henrik Boström91aa7322020-04-28 12:24:33 +02002476 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002477 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002478 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002479 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2480 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002481 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2482 last_wants.max_pixel_count ||
2483 video_source_.sink_wants().max_framerate_fps >
2484 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002485 }
2486
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002487 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002488 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002489 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002490 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2491 EXPECT_EQ((loop_count - 1) * 2,
2492 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002493
mflodmancc3d4422017-08-03 08:27:51 -07002494 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002495}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002496
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002497TEST_F(VideoStreamEncoderTest,
2498 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2499 video_stream_encoder_->OnBitrateUpdated(
2500 DataRate::BitsPerSec(kTargetBitrateBps),
2501 DataRate::BitsPerSec(kTargetBitrateBps),
2502 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002503 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002504
2505 const int kFrameWidth = 1280;
2506 const int kFrameHeight = 720;
2507
2508 int64_t ntp_time = kFrameIntervalMs;
2509
2510 // Force an input frame rate to be available, or the adaptation call won't
2511 // know what framerate to adapt form.
2512 const int kInputFps = 30;
2513 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2514 stats.input_frame_rate = kInputFps;
2515 stats_proxy_->SetMockStats(stats);
2516
2517 video_source_.set_adaptation_enabled(true);
2518 video_stream_encoder_->SetSource(
2519 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002520 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002521 video_source_.IncomingCapturedFrame(
2522 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2523 sink_.WaitForEncodedFrame(ntp_time);
2524 ntp_time += kFrameIntervalMs;
2525
2526 // Trigger CPU overuse.
2527 video_stream_encoder_->TriggerCpuOveruse();
2528 video_source_.IncomingCapturedFrame(
2529 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2530 sink_.WaitForEncodedFrame(ntp_time);
2531 ntp_time += kFrameIntervalMs;
2532
2533 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2534 EXPECT_EQ(std::numeric_limits<int>::max(),
2535 video_source_.sink_wants().max_pixel_count);
2536 // Some framerate constraint should be set.
2537 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2538 EXPECT_LT(restricted_fps, kInputFps);
2539 video_source_.IncomingCapturedFrame(
2540 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2541 sink_.WaitForEncodedFrame(ntp_time);
2542 ntp_time += 100;
2543
Henrik Boström2671dac2020-05-19 16:29:09 +02002544 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002545 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2546 // Give the encoder queue time to process the change in degradation preference
2547 // by waiting for an encoded frame.
2548 video_source_.IncomingCapturedFrame(
2549 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2550 sink_.WaitForEncodedFrame(ntp_time);
2551 ntp_time += kFrameIntervalMs;
2552
2553 video_stream_encoder_->TriggerQualityLow();
2554 video_source_.IncomingCapturedFrame(
2555 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2556 sink_.WaitForEncodedFrame(ntp_time);
2557 ntp_time += kFrameIntervalMs;
2558
2559 // Some resolution constraint should be set.
2560 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2561 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2562 kFrameWidth * kFrameHeight);
2563 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2564
2565 int pixel_count = video_source_.sink_wants().max_pixel_count;
2566 // Triggering a CPU underuse should not change the sink wants since it has
2567 // not been overused for resolution since we changed degradation preference.
2568 video_stream_encoder_->TriggerCpuUnderuse();
2569 video_source_.IncomingCapturedFrame(
2570 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2571 sink_.WaitForEncodedFrame(ntp_time);
2572 ntp_time += kFrameIntervalMs;
2573 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2574 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2575
Evan Shrubsole64469032020-06-11 10:45:29 +02002576 // Change the degradation preference back. CPU underuse should not adapt since
2577 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002578 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002579 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2580 video_source_.IncomingCapturedFrame(
2581 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2582 sink_.WaitForEncodedFrame(ntp_time);
2583 ntp_time += 100;
2584 // Resolution adaptations is gone after changing degradation preference.
2585 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2586 EXPECT_EQ(std::numeric_limits<int>::max(),
2587 video_source_.sink_wants().max_pixel_count);
2588 // The fps adaptation from above is now back.
2589 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2590
2591 // Trigger CPU underuse.
2592 video_stream_encoder_->TriggerCpuUnderuse();
2593 video_source_.IncomingCapturedFrame(
2594 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2595 sink_.WaitForEncodedFrame(ntp_time);
2596 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002597 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2598
2599 // Trigger QP underuse, fps should return to normal.
2600 video_stream_encoder_->TriggerQualityHigh();
2601 video_source_.IncomingCapturedFrame(
2602 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2603 sink_.WaitForEncodedFrame(ntp_time);
2604 ntp_time += kFrameIntervalMs;
2605 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002606
2607 video_stream_encoder_->Stop();
2608}
2609
mflodmancc3d4422017-08-03 08:27:51 -07002610TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002612 DataRate::BitsPerSec(kTargetBitrateBps),
2613 DataRate::BitsPerSec(kTargetBitrateBps),
2614 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002615 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002616
sprangc5d62e22017-04-02 23:53:04 -07002617 const int kFrameWidth = 1280;
2618 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002619
Åsa Persson8c1bf952018-09-13 10:42:19 +02002620 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002621
kthelgason5e13d412016-12-01 03:59:51 -08002622 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002623 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002624 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002625 frame_timestamp += kFrameIntervalMs;
2626
perkj803d97f2016-11-01 11:45:46 -07002627 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002628 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002629 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002630 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002631 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002632 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002633
asapersson0944a802017-04-07 00:57:58 -07002634 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002635 // wanted resolution.
2636 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2637 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2638 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002639 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002640
2641 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002642 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002643 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002644 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002645 // Give the encoder queue time to process the change in degradation preference
2646 // by waiting for an encoded frame.
2647 new_video_source.IncomingCapturedFrame(
2648 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2649 sink_.WaitForEncodedFrame(frame_timestamp);
2650 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002651 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002652 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002653
sprangc5d62e22017-04-02 23:53:04 -07002654 // Force an input frame rate to be available, or the adaptation call won't
2655 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002656 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002657 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002658 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002659 stats_proxy_->SetMockStats(stats);
2660
mflodmancc3d4422017-08-03 08:27:51 -07002661 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002662 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002663 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002664 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002665 frame_timestamp += kFrameIntervalMs;
2666
2667 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002668 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002669 EXPECT_EQ(std::numeric_limits<int>::max(),
2670 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002671 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002672
asapersson02465b82017-04-10 01:12:52 -07002673 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002674 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2675 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002676 // Give the encoder queue time to process the change in degradation preference
2677 // by waiting for an encoded frame.
2678 new_video_source.IncomingCapturedFrame(
2679 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2680 sink_.WaitForEncodedFrame(frame_timestamp);
2681 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002682 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002683
mflodmancc3d4422017-08-03 08:27:51 -07002684 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002685 new_video_source.IncomingCapturedFrame(
2686 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002687 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002688 frame_timestamp += kFrameIntervalMs;
2689
2690 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002691 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002692
2693 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002694 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002695 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002696 // Give the encoder queue time to process the change in degradation preference
2697 // by waiting for an encoded frame.
2698 new_video_source.IncomingCapturedFrame(
2699 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2700 sink_.WaitForEncodedFrame(frame_timestamp);
2701 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002702 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2703 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002704 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002705 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002706
2707 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002708 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002709 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002710 // Give the encoder queue time to process the change in degradation preference
2711 // by waiting for an encoded frame.
2712 new_video_source.IncomingCapturedFrame(
2713 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2714 sink_.WaitForEncodedFrame(frame_timestamp);
2715 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002716 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2717 EXPECT_EQ(std::numeric_limits<int>::max(),
2718 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002719 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002720
mflodmancc3d4422017-08-03 08:27:51 -07002721 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002722}
2723
mflodmancc3d4422017-08-03 08:27:51 -07002724TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002726 DataRate::BitsPerSec(kTargetBitrateBps),
2727 DataRate::BitsPerSec(kTargetBitrateBps),
2728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002729
asaperssonfab67072017-04-04 05:51:49 -07002730 const int kWidth = 1280;
2731 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002732 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002733 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002734 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2735 EXPECT_FALSE(stats.bw_limited_resolution);
2736 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2737
2738 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002739 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002740 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002741 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002742
2743 stats = stats_proxy_->GetStats();
2744 EXPECT_TRUE(stats.bw_limited_resolution);
2745 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2746
2747 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002748 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002749 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002750 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002751
2752 stats = stats_proxy_->GetStats();
2753 EXPECT_FALSE(stats.bw_limited_resolution);
2754 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2755 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2756
mflodmancc3d4422017-08-03 08:27:51 -07002757 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002758}
2759
mflodmancc3d4422017-08-03 08:27:51 -07002760TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002762 DataRate::BitsPerSec(kTargetBitrateBps),
2763 DataRate::BitsPerSec(kTargetBitrateBps),
2764 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002765
2766 const int kWidth = 1280;
2767 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002768 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002769 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002770 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2771 EXPECT_FALSE(stats.cpu_limited_resolution);
2772 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2773
2774 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002775 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002776 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002777 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002778
2779 stats = stats_proxy_->GetStats();
2780 EXPECT_TRUE(stats.cpu_limited_resolution);
2781 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2782
2783 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002784 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002785 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002786 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002787
2788 stats = stats_proxy_->GetStats();
2789 EXPECT_FALSE(stats.cpu_limited_resolution);
2790 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002791 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002792
mflodmancc3d4422017-08-03 08:27:51 -07002793 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002794}
2795
mflodmancc3d4422017-08-03 08:27:51 -07002796TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002798 DataRate::BitsPerSec(kTargetBitrateBps),
2799 DataRate::BitsPerSec(kTargetBitrateBps),
2800 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002801
asaperssonfab67072017-04-04 05:51:49 -07002802 const int kWidth = 1280;
2803 const int kHeight = 720;
2804 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002805 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002806 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002807 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002808 EXPECT_FALSE(stats.cpu_limited_resolution);
2809 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2810
asaperssonfab67072017-04-04 05:51:49 -07002811 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002812 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002813 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002814 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002815 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002816 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002817 EXPECT_TRUE(stats.cpu_limited_resolution);
2818 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2819
2820 // Set new source with adaptation still enabled.
2821 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002822 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002823 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002824
asaperssonfab67072017-04-04 05:51:49 -07002825 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002826 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002827 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002828 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002829 EXPECT_TRUE(stats.cpu_limited_resolution);
2830 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2831
2832 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002833 video_stream_encoder_->SetSource(&new_video_source,
2834 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002835
asaperssonfab67072017-04-04 05:51:49 -07002836 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002837 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002838 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002839 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002840 EXPECT_FALSE(stats.cpu_limited_resolution);
2841 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2842
2843 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002844 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002845 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002846
asaperssonfab67072017-04-04 05:51:49 -07002847 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002848 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002849 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002850 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002851 EXPECT_TRUE(stats.cpu_limited_resolution);
2852 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2853
asaperssonfab67072017-04-04 05:51:49 -07002854 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002855 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002856 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002857 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002858 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002859 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002860 EXPECT_FALSE(stats.cpu_limited_resolution);
2861 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002862 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002863
mflodmancc3d4422017-08-03 08:27:51 -07002864 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002865}
2866
mflodmancc3d4422017-08-03 08:27:51 -07002867TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002869 DataRate::BitsPerSec(kTargetBitrateBps),
2870 DataRate::BitsPerSec(kTargetBitrateBps),
2871 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002872
asaperssonfab67072017-04-04 05:51:49 -07002873 const int kWidth = 1280;
2874 const int kHeight = 720;
2875 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002876 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002877 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002878 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002879 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002880 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002881
2882 // Set new source with adaptation still enabled.
2883 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002884 video_stream_encoder_->SetSource(&new_video_source,
2885 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002886
asaperssonfab67072017-04-04 05:51:49 -07002887 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002888 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002889 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002890 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002891 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002892 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002893
asaperssonfab67072017-04-04 05:51:49 -07002894 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002895 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002896 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002897 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002898 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002899 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002900 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002901 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002902
asaperssonfab67072017-04-04 05:51:49 -07002903 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002904 video_stream_encoder_->SetSource(&new_video_source,
2905 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002906
asaperssonfab67072017-04-04 05:51:49 -07002907 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002908 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002909 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002910 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002911 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002912 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002913
asapersson02465b82017-04-10 01:12:52 -07002914 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002915 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002916 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002917
asaperssonfab67072017-04-04 05:51:49 -07002918 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002919 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002920 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002921 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002922 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002923 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2924 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002925
mflodmancc3d4422017-08-03 08:27:51 -07002926 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002927}
2928
mflodmancc3d4422017-08-03 08:27:51 -07002929TEST_F(VideoStreamEncoderTest,
2930 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002932 DataRate::BitsPerSec(kTargetBitrateBps),
2933 DataRate::BitsPerSec(kTargetBitrateBps),
2934 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002935
2936 const int kWidth = 1280;
2937 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002938 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002939 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002940 video_source_.IncomingCapturedFrame(
2941 CreateFrame(timestamp_ms, kWidth, kHeight));
2942 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002943 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2944 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2945 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2946
2947 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002948 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002949 timestamp_ms += kFrameIntervalMs;
2950 video_source_.IncomingCapturedFrame(
2951 CreateFrame(timestamp_ms, kWidth, kHeight));
2952 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002953 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2954 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2955 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2956
2957 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002958 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002959 timestamp_ms += kFrameIntervalMs;
2960 video_source_.IncomingCapturedFrame(
2961 CreateFrame(timestamp_ms, kWidth, kHeight));
2962 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002963 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2964 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2965 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2966
Niels Möller4db138e2018-04-19 09:04:13 +02002967 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002968 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002969
2970 VideoEncoderConfig video_encoder_config;
2971 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2972 // Make format different, to force recreation of encoder.
2973 video_encoder_config.video_format.parameters["foo"] = "foo";
2974 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002975 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002976 timestamp_ms += kFrameIntervalMs;
2977 video_source_.IncomingCapturedFrame(
2978 CreateFrame(timestamp_ms, kWidth, kHeight));
2979 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002980 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2981 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2982 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2983
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002985}
2986
mflodmancc3d4422017-08-03 08:27:51 -07002987TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002988 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002990 DataRate::BitsPerSec(kTargetBitrateBps),
2991 DataRate::BitsPerSec(kTargetBitrateBps),
2992 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2993
2994 const int kWidth = 1280;
2995 const int kHeight = 720;
2996 int sequence = 1;
2997
2998 // Enable BALANCED preference, no initial limitation.
2999 test::FrameForwarder source;
3000 video_stream_encoder_->SetSource(&source,
3001 webrtc::DegradationPreference::BALANCED);
3002 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3003 WaitForEncodedFrame(sequence++);
3004 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3005 EXPECT_FALSE(stats.cpu_limited_resolution);
3006 EXPECT_FALSE(stats.cpu_limited_framerate);
3007 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3008
3009 // Trigger CPU overuse, should now adapt down.
3010 video_stream_encoder_->TriggerCpuOveruse();
3011 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3012 WaitForEncodedFrame(sequence++);
3013 stats = stats_proxy_->GetStats();
3014 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3015
3016 // Set new degradation preference should clear restrictions since we changed
3017 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003018 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003019 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3020 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3021 WaitForEncodedFrame(sequence++);
3022 stats = stats_proxy_->GetStats();
3023 EXPECT_FALSE(stats.cpu_limited_resolution);
3024 EXPECT_FALSE(stats.cpu_limited_framerate);
3025 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3026
3027 // Force an input frame rate to be available, or the adaptation call won't
3028 // know what framerate to adapt from.
3029 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3030 mock_stats.input_frame_rate = 30;
3031 stats_proxy_->SetMockStats(mock_stats);
3032 video_stream_encoder_->TriggerCpuOveruse();
3033 stats_proxy_->ResetMockStats();
3034 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3035 WaitForEncodedFrame(sequence++);
3036
3037 // We have now adapted once.
3038 stats = stats_proxy_->GetStats();
3039 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3040
3041 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003042 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3043 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003044 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3045 WaitForEncodedFrame(sequence++);
3046 stats = stats_proxy_->GetStats();
3047 EXPECT_FALSE(stats.cpu_limited_resolution);
3048 EXPECT_FALSE(stats.cpu_limited_framerate);
3049 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3050
3051 video_stream_encoder_->Stop();
3052}
3053
3054TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003055 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003057 DataRate::BitsPerSec(kTargetBitrateBps),
3058 DataRate::BitsPerSec(kTargetBitrateBps),
3059 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003060
asapersson0944a802017-04-07 00:57:58 -07003061 const int kWidth = 1280;
3062 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003063 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003064
asaperssonfab67072017-04-04 05:51:49 -07003065 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003066 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003067 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003068 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003069 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003070 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3071
asapersson02465b82017-04-10 01:12:52 -07003072 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003073 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003074 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003075 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003076 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003077 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003078 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003079 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3080
3081 // Set new source with adaptation still enabled.
3082 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003083 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003084 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003085
3086 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003087 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003088 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003089 stats = stats_proxy_->GetStats();
3090 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003091 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003092 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3093
sprangc5d62e22017-04-02 23:53:04 -07003094 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003096 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003097 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003098 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003099 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003100 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003101 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003102 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003103 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003104 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3105
sprangc5d62e22017-04-02 23:53:04 -07003106 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003107 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003108 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3109 mock_stats.input_frame_rate = 30;
3110 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003111 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003112 stats_proxy_->ResetMockStats();
3113
3114 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003115 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003116 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003117
3118 // Framerate now adapted.
3119 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003120 EXPECT_FALSE(stats.cpu_limited_resolution);
3121 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003122 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3123
3124 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003125 video_stream_encoder_->SetSource(&new_video_source,
3126 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003127 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003128 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003129 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003130
3131 stats = stats_proxy_->GetStats();
3132 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003133 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003134 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3135
3136 // Try to trigger overuse. Should not succeed.
3137 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003138 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003139 stats_proxy_->ResetMockStats();
3140
3141 stats = stats_proxy_->GetStats();
3142 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003143 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003144 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3145
3146 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003147 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003148 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003149 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003150 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003151 stats = stats_proxy_->GetStats();
3152 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003153 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003154 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003155
3156 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003157 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003158 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003159 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003160 stats = stats_proxy_->GetStats();
3161 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003162 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003163 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3164
3165 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003166 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003167 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003168 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003169 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003170 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003171 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003172 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003173 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003174 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003175 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3176
3177 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003178 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003179 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003180 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003181 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003182 stats = stats_proxy_->GetStats();
3183 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003184 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003185 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003186 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003187
mflodmancc3d4422017-08-03 08:27:51 -07003188 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003189}
3190
mflodmancc3d4422017-08-03 08:27:51 -07003191TEST_F(VideoStreamEncoderTest,
3192 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003193 const int kWidth = 1280;
3194 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003196 DataRate::BitsPerSec(kTargetBitrateBps),
3197 DataRate::BitsPerSec(kTargetBitrateBps),
3198 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003199
asaperssonfab67072017-04-04 05:51:49 -07003200 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003201 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003202
asaperssonfab67072017-04-04 05:51:49 -07003203 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003204 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003205
asaperssonfab67072017-04-04 05:51:49 -07003206 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003208
asaperssonfab67072017-04-04 05:51:49 -07003209 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003210 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003211
kthelgason876222f2016-11-29 01:44:11 -08003212 // Expect a scale down.
3213 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003214 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003215
asapersson02465b82017-04-10 01:12:52 -07003216 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003217 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003218 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003219 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003220
asaperssonfab67072017-04-04 05:51:49 -07003221 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003222 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003223 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003224 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003225
asaperssonfab67072017-04-04 05:51:49 -07003226 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003227 EXPECT_EQ(std::numeric_limits<int>::max(),
3228 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003229
asaperssonfab67072017-04-04 05:51:49 -07003230 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003231 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003232 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003233 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003234
asapersson02465b82017-04-10 01:12:52 -07003235 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003236 EXPECT_EQ(std::numeric_limits<int>::max(),
3237 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003238
mflodmancc3d4422017-08-03 08:27:51 -07003239 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003240}
3241
mflodmancc3d4422017-08-03 08:27:51 -07003242TEST_F(VideoStreamEncoderTest,
3243 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003244 const int kWidth = 1280;
3245 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003246 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003247 DataRate::BitsPerSec(kTargetBitrateBps),
3248 DataRate::BitsPerSec(kTargetBitrateBps),
3249 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003250
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003251 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003252 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003253 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003254 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003255
3256 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003257 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003258 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003259 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3260 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3261
3262 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003263 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003264 EXPECT_THAT(source.sink_wants(),
3265 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003266 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3267 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3268 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3269
3270 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003271 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003272 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3273 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3274 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3275
mflodmancc3d4422017-08-03 08:27:51 -07003276 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003277}
3278
mflodmancc3d4422017-08-03 08:27:51 -07003279TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003280 const int kWidth = 1280;
3281 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003282 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003283 DataRate::BitsPerSec(kTargetBitrateBps),
3284 DataRate::BitsPerSec(kTargetBitrateBps),
3285 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003286
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003287 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003288 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003289 video_stream_encoder_->SetSource(&source,
3290 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003291 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3292 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003293 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003294
3295 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003296 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003297 EXPECT_THAT(source.sink_wants(),
3298 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003299 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3300 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3301 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3302
3303 // Trigger adapt down for same input resolution, expect no change.
3304 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3305 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003306 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003307 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3308 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3310
3311 // Trigger adapt down for larger input resolution, expect no change.
3312 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3313 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003314 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003315 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3317 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3318
mflodmancc3d4422017-08-03 08:27:51 -07003319 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003320}
3321
mflodmancc3d4422017-08-03 08:27:51 -07003322TEST_F(VideoStreamEncoderTest,
3323 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003324 const int kWidth = 1280;
3325 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003327 DataRate::BitsPerSec(kTargetBitrateBps),
3328 DataRate::BitsPerSec(kTargetBitrateBps),
3329 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003330
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003331 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003332 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003334 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003335
3336 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003337 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003338 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3341
3342 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003343 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003344 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003345 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3347
mflodmancc3d4422017-08-03 08:27:51 -07003348 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003349}
3350
mflodmancc3d4422017-08-03 08:27:51 -07003351TEST_F(VideoStreamEncoderTest,
3352 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003353 const int kWidth = 1280;
3354 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003356 DataRate::BitsPerSec(kTargetBitrateBps),
3357 DataRate::BitsPerSec(kTargetBitrateBps),
3358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003359
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003360 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003361 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003362 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003363 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003364
3365 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003366 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003367 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003368 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003369 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3370
3371 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003372 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003373 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003374 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003375 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3376
mflodmancc3d4422017-08-03 08:27:51 -07003377 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003378}
3379
mflodmancc3d4422017-08-03 08:27:51 -07003380TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003381 const int kWidth = 1280;
3382 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003384 DataRate::BitsPerSec(kTargetBitrateBps),
3385 DataRate::BitsPerSec(kTargetBitrateBps),
3386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003387
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003388 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003389 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003390 video_stream_encoder_->SetSource(&source,
3391 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003392
3393 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3394 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003395 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003396 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3397 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3398 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003401 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003402 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3404 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
mflodmancc3d4422017-08-03 08:27:51 -07003407 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003408}
3409
mflodmancc3d4422017-08-03 08:27:51 -07003410TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003411 const int kWidth = 1280;
3412 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003414 DataRate::BitsPerSec(kTargetBitrateBps),
3415 DataRate::BitsPerSec(kTargetBitrateBps),
3416 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003417
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003418 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003419 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003420 video_stream_encoder_->SetSource(&source,
3421 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003422
3423 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3424 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003425 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3427 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3428 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
3430 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003431 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003432 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3435 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
mflodmancc3d4422017-08-03 08:27:51 -07003437 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003438}
3439
mflodmancc3d4422017-08-03 08:27:51 -07003440TEST_F(VideoStreamEncoderTest,
3441 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003442 const int kWidth = 1280;
3443 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003444 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003445 DataRate::BitsPerSec(kTargetBitrateBps),
3446 DataRate::BitsPerSec(kTargetBitrateBps),
3447 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003448
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003449 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003450 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003451 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003452 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003453 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003454
3455 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003456 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003457 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3459 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003462 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003463 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003464 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003465 EXPECT_THAT(source.sink_wants(),
3466 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003467 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3468 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3469
3470 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003471 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003472 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3474 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3476
mflodmancc3d4422017-08-03 08:27:51 -07003477 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003478}
3479
mflodmancc3d4422017-08-03 08:27:51 -07003480TEST_F(VideoStreamEncoderTest,
3481 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003482 const int kWidth = 1280;
3483 const int kHeight = 720;
3484 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003486 DataRate::BitsPerSec(kTargetBitrateBps),
3487 DataRate::BitsPerSec(kTargetBitrateBps),
3488 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003489
3490 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3491 stats.input_frame_rate = kInputFps;
3492 stats_proxy_->SetMockStats(stats);
3493
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003494 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003495 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3496 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003497 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003498
3499 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003500 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003501 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3502 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003503 EXPECT_THAT(video_source_.sink_wants(),
3504 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003505
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003506 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003507 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003508 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003509 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003510 // Give the encoder queue time to process the change in degradation preference
3511 // by waiting for an encoded frame.
3512 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3513 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003514 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003515
3516 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003517 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003518 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3519 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003520 EXPECT_THAT(new_video_source.sink_wants(),
3521 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003522
3523 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003524 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003525 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003526
mflodmancc3d4422017-08-03 08:27:51 -07003527 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003528}
3529
mflodmancc3d4422017-08-03 08:27:51 -07003530TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003531 const int kWidth = 1280;
3532 const int kHeight = 720;
3533 const size_t kNumFrames = 10;
3534
Henrik Boström381d1092020-05-12 18:49:07 +02003535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003536 DataRate::BitsPerSec(kTargetBitrateBps),
3537 DataRate::BitsPerSec(kTargetBitrateBps),
3538 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003539
asaperssond0de2952017-04-21 01:47:31 -07003540 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003541 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003542 video_source_.set_adaptation_enabled(true);
3543
3544 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3545 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3546
3547 int downscales = 0;
3548 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003549 video_source_.IncomingCapturedFrame(
3550 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3551 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003552
asaperssonfab67072017-04-04 05:51:49 -07003553 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003554 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003555 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003556 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003557
3558 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3559 ++downscales;
3560
3561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_EQ(downscales,
3563 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003565 }
mflodmancc3d4422017-08-03 08:27:51 -07003566 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003567}
3568
mflodmancc3d4422017-08-03 08:27:51 -07003569TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003570 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3571 const int kWidth = 1280;
3572 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003573 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003574 DataRate::BitsPerSec(kTargetBitrateBps),
3575 DataRate::BitsPerSec(kTargetBitrateBps),
3576 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003577
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003578 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003579 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003580 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003581 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003582 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003583
Åsa Persson8c1bf952018-09-13 10:42:19 +02003584 int64_t timestamp_ms = kFrameIntervalMs;
3585 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003586 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003587 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003588 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3589 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3590
3591 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003592 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003593 timestamp_ms += kFrameIntervalMs;
3594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003596 EXPECT_THAT(source.sink_wants(),
3597 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003598 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3599 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3600
3601 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003602 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003603 timestamp_ms += kFrameIntervalMs;
3604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003605 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003606 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3608 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3609
3610 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003611 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003612 timestamp_ms += kFrameIntervalMs;
3613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003615 EXPECT_THAT(source.sink_wants(),
3616 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003617 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3618 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3619
3620 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003621 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003622 timestamp_ms += kFrameIntervalMs;
3623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003624 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003625 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3627 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3628
mflodmancc3d4422017-08-03 08:27:51 -07003629 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003630}
3631
mflodmancc3d4422017-08-03 08:27:51 -07003632TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003633 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3634 const int kWidth = 1280;
3635 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003637 DataRate::BitsPerSec(kTargetBitrateBps),
3638 DataRate::BitsPerSec(kTargetBitrateBps),
3639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003640
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003641 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003642 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003643 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003644 video_stream_encoder_->SetSource(&source,
3645 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003646
Åsa Persson8c1bf952018-09-13 10:42:19 +02003647 int64_t timestamp_ms = kFrameIntervalMs;
3648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003649 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003650 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3652 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3653
3654 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003655 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003656 timestamp_ms += kFrameIntervalMs;
3657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3658 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003659 EXPECT_THAT(source.sink_wants(),
3660 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3662 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3663
3664 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003665 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003666 timestamp_ms += kFrameIntervalMs;
3667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003668 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003669 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3671 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3672
3673 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003674 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003675 timestamp_ms += kFrameIntervalMs;
3676 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3677 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003678 EXPECT_THAT(source.sink_wants(),
3679 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003680 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3681 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3682
3683 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003684 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003685 timestamp_ms += kFrameIntervalMs;
3686 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003687 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3690 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3691
mflodmancc3d4422017-08-03 08:27:51 -07003692 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003693}
3694
Sergey Silkin41c650b2019-10-14 13:12:19 +02003695TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3696 fake_encoder_.SetResolutionBitrateLimits(
3697 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3698
Henrik Boström381d1092020-05-12 18:49:07 +02003699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003700 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3701 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3702 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3703 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003704
3705 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003706 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003707 source.set_adaptation_enabled(true);
3708 video_stream_encoder_->SetSource(
3709 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3710
3711 // Insert 720p frame.
3712 int64_t timestamp_ms = kFrameIntervalMs;
3713 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3714 WaitForEncodedFrame(1280, 720);
3715
3716 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003718 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3719 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3720 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3721 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003722 video_stream_encoder_->TriggerQualityLow();
3723
3724 // Insert 720p frame. It should be downscaled and encoded.
3725 timestamp_ms += kFrameIntervalMs;
3726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3727 WaitForEncodedFrame(960, 540);
3728
3729 // Trigger adapt up. Higher resolution should not be requested duo to lack
3730 // of bitrate.
3731 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003732 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003733
3734 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003736 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3737 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3738 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3739 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003740
3741 // Trigger adapt up. Higher resolution should be requested.
3742 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003743 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003744
3745 video_stream_encoder_->Stop();
3746}
3747
3748TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3749 fake_encoder_.SetResolutionBitrateLimits(
3750 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3751
3752 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003754 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3755 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3756 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3757 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003758
3759 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003760 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003761 source.set_adaptation_enabled(true);
3762 video_stream_encoder_->SetSource(
3763 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3764
3765 // Insert 720p frame. It should be dropped and lower resolution should be
3766 // requested.
3767 int64_t timestamp_ms = kFrameIntervalMs;
3768 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3769 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003770 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003771
3772 // Insert 720p frame. It should be downscaled and encoded.
3773 timestamp_ms += kFrameIntervalMs;
3774 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3775 WaitForEncodedFrame(960, 540);
3776
3777 video_stream_encoder_->Stop();
3778}
3779
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003780class BalancedDegradationTest : public VideoStreamEncoderTest {
3781 protected:
3782 void SetupTest() {
3783 // Reset encoder for field trials to take effect.
3784 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003785 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003786
3787 // Enable BALANCED preference.
3788 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003789 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3790 }
3791
3792 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003793 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003794 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3795 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003796 }
3797
Åsa Persson45b176f2019-09-30 11:19:05 +02003798 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003799 timestamp_ms_ += kFrameIntervalMs;
3800 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003801 }
3802
3803 void InsertFrameAndWaitForEncoded() {
3804 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003805 sink_.WaitForEncodedFrame(timestamp_ms_);
3806 }
3807
3808 const int kWidth = 640; // pixels:640x360=230400
3809 const int kHeight = 360;
3810 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3811 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003812 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003813};
3814
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003815TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003816 test::ScopedFieldTrials field_trials(
3817 "WebRTC-Video-BalancedDegradationSettings/"
3818 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3819 SetupTest();
3820
3821 // Force input frame rate.
3822 const int kInputFps = 24;
3823 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3824 stats.input_frame_rate = kInputFps;
3825 stats_proxy_->SetMockStats(stats);
3826
Åsa Persson45b176f2019-09-30 11:19:05 +02003827 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003828 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003829
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003830 // Trigger adapt down, expect scaled down framerate and resolution,
3831 // since Fps diff (input-requested:0) < threshold.
3832 video_stream_encoder_->TriggerQualityLow();
3833 EXPECT_THAT(source_.sink_wants(),
3834 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003835
3836 video_stream_encoder_->Stop();
3837}
3838
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003839TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003840 test::ScopedFieldTrials field_trials(
3841 "WebRTC-Video-BalancedDegradationSettings/"
3842 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3843 SetupTest();
3844
3845 // Force input frame rate.
3846 const int kInputFps = 25;
3847 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3848 stats.input_frame_rate = kInputFps;
3849 stats_proxy_->SetMockStats(stats);
3850
Åsa Persson45b176f2019-09-30 11:19:05 +02003851 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003852 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003853
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003854 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3855 // Fps diff (input-requested:1) == threshold.
3856 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003857 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003858
3859 video_stream_encoder_->Stop();
3860}
3861
3862TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3863 test::ScopedFieldTrials field_trials(
3864 "WebRTC-Video-BalancedDegradationSettings/"
3865 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3866 SetupTest();
3867
3868 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3869
Åsa Persson45b176f2019-09-30 11:19:05 +02003870 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003871 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003872
3873 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3874 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003875 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003876
3877 video_stream_encoder_->Stop();
3878}
3879
Åsa Perssonccfb3402019-09-25 15:13:04 +02003880TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003881 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003882 "WebRTC-Video-BalancedDegradationSettings/"
3883 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003884 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003885
Åsa Persson1b247f12019-08-14 17:26:39 +02003886 const int kMinBitrateBps = 425000;
3887 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003888 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003889
Åsa Persson45b176f2019-09-30 11:19:05 +02003890 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003891 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003892 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3893
3894 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3895 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003896 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003897 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003898 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3899
3900 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3901 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003902 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003903 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003904 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3905
Åsa Persson30ab0152019-08-27 12:22:33 +02003906 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3907 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003908 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003909 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003910 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003911 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3912
3913 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003914 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003915 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003916 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003917
Åsa Persson30ab0152019-08-27 12:22:33 +02003918 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003919 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003920 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003921 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003922 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003923 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3924
3925 video_stream_encoder_->Stop();
3926}
3927
Åsa Perssonccfb3402019-09-25 15:13:04 +02003928TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003929 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3930 test::ScopedFieldTrials field_trials(
3931 "WebRTC-Video-BalancedDegradationSettings/"
3932 "pixels:57600|129600|230400,fps:7|24|24/");
3933 SetupTest();
3934 OnBitrateUpdated(kLowTargetBitrateBps);
3935
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003936 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003937
3938 // Insert frame, expect scaled down:
3939 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3940 InsertFrame();
3941 EXPECT_FALSE(WaitForFrame(1000));
3942 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3943 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3944
3945 // Insert frame, expect scaled down:
3946 // resolution (320x180@24fps).
3947 InsertFrame();
3948 EXPECT_FALSE(WaitForFrame(1000));
3949 EXPECT_LT(source_.sink_wants().max_pixel_count,
3950 source_.last_wants().max_pixel_count);
3951 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3952
3953 // Frame should not be dropped (min pixels per frame reached).
3954 InsertFrameAndWaitForEncoded();
3955
3956 video_stream_encoder_->Stop();
3957}
3958
3959TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003960 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003961 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003962 "WebRTC-Video-BalancedDegradationSettings/"
3963 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003964 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003965
Åsa Persson30ab0152019-08-27 12:22:33 +02003966 const int kResolutionMinBitrateBps = 435000;
3967 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003968 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003969
Åsa Persson45b176f2019-09-30 11:19:05 +02003970 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003971 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003972 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3973
3974 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3975 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003976 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003977 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003978 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3979
3980 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3981 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003982 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003983 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003984 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3985
3986 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3987 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003988 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003989 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003990 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3991
Åsa Persson30ab0152019-08-27 12:22:33 +02003992 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3993 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003994 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003995 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003996 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3997
3998 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3999 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004000 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004001 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4002
4003 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004004 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004005 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004006 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004007 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004008 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4009
4010 video_stream_encoder_->Stop();
4011}
4012
Åsa Perssonccfb3402019-09-25 15:13:04 +02004013TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004014 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004015 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004016 "WebRTC-Video-BalancedDegradationSettings/"
4017 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004018 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004019
Åsa Persson30ab0152019-08-27 12:22:33 +02004020 const int kMinBitrateBps = 425000;
4021 const int kTooLowMinBitrateBps = 424000;
4022 const int kResolutionMinBitrateBps = 435000;
4023 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004024 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004025
Åsa Persson45b176f2019-09-30 11:19:05 +02004026 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004027 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4029
4030 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4031 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004032 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004033 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004034 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4035
4036 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4037 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004038 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004039 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004040 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4041
4042 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4043 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004044 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004045 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004046 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4047
4048 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4049 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004050 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004051 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4052
4053 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004054 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004055 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004056 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004057 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004058 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4059
4060 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004061 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004062 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004063 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004064 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4065
4066 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004067 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004068 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004069 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004070 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004071 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4072
Åsa Persson1b247f12019-08-14 17:26:39 +02004073 video_stream_encoder_->Stop();
4074}
4075
mflodmancc3d4422017-08-03 08:27:51 -07004076TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004077 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4078 const int kWidth = 1280;
4079 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004081 DataRate::BitsPerSec(kTargetBitrateBps),
4082 DataRate::BitsPerSec(kTargetBitrateBps),
4083 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004084
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004085 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004086 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004087 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004088 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004089 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004090
Åsa Persson8c1bf952018-09-13 10:42:19 +02004091 int64_t timestamp_ms = kFrameIntervalMs;
4092 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004093 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004094 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004095 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4096 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4097 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4098 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4099
4100 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004101 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004102 timestamp_ms += kFrameIntervalMs;
4103 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4104 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004105 EXPECT_THAT(source.sink_wants(),
4106 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004107 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4108 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4109 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4110 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4111
4112 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004113 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004114 timestamp_ms += kFrameIntervalMs;
4115 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4116 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004117 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004118 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4119 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4120 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4121 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4122
Jonathan Yubc771b72017-12-08 17:04:29 -08004123 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004124 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004125 timestamp_ms += kFrameIntervalMs;
4126 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4127 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004128 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004129 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4130 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004131 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004132 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4133
Jonathan Yubc771b72017-12-08 17:04:29 -08004134 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004135 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004136 timestamp_ms += kFrameIntervalMs;
4137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4138 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004139 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004140 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004141 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4142 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4143 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4144 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4145
Jonathan Yubc771b72017-12-08 17:04:29 -08004146 // Trigger quality adapt down, expect no change (min resolution reached).
4147 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004148 timestamp_ms += kFrameIntervalMs;
4149 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4150 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004151 EXPECT_THAT(source.sink_wants(), FpsMax());
4152 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004153 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4154 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4155 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4156 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4157
Evan Shrubsole64469032020-06-11 10:45:29 +02004158 // Trigger quality adapt up, expect upscaled resolution (480x270).
4159 video_stream_encoder_->TriggerQualityHigh();
4160 timestamp_ms += kFrameIntervalMs;
4161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4162 WaitForEncodedFrame(timestamp_ms);
4163 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4164 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4165 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4166 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4167 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4168
4169 // Trigger quality and cpu adapt up since both are most limited, expect
4170 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004171 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004172 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004173 timestamp_ms += kFrameIntervalMs;
4174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4175 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004176 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004177 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4179 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004180 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004181
Evan Shrubsole64469032020-06-11 10:45:29 +02004182 // Trigger quality and cpu adapt up since both are most limited, expect
4183 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004184 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004185 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004186 timestamp_ms += kFrameIntervalMs;
4187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4188 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004189 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004190 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004191 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004192 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004193 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4194 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004195
Evan Shrubsole64469032020-06-11 10:45:29 +02004196 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4197 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004198 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004199 timestamp_ms += kFrameIntervalMs;
4200 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4201 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004202 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004203 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4204 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004205 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004206 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004207
4208 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004209 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004210 timestamp_ms += kFrameIntervalMs;
4211 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004212 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004213 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004214 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004215 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4216 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004217 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004218 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004219
mflodmancc3d4422017-08-03 08:27:51 -07004220 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004221}
4222
mflodmancc3d4422017-08-03 08:27:51 -07004223TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004224 const int kWidth = 640;
4225 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004226
Henrik Boström381d1092020-05-12 18:49:07 +02004227 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004228 DataRate::BitsPerSec(kTargetBitrateBps),
4229 DataRate::BitsPerSec(kTargetBitrateBps),
4230 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004231
perkj803d97f2016-11-01 11:45:46 -07004232 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004233 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004234 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004235 }
4236
mflodmancc3d4422017-08-03 08:27:51 -07004237 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004238 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004239 video_source_.IncomingCapturedFrame(CreateFrame(
4240 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004241 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004242 }
4243
mflodmancc3d4422017-08-03 08:27:51 -07004244 video_stream_encoder_->Stop();
4245 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004246 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004247
Ying Wangef3998f2019-12-09 13:06:53 +01004248 EXPECT_METRIC_EQ(
4249 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4250 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004251 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4252}
4253
mflodmancc3d4422017-08-03 08:27:51 -07004254TEST_F(VideoStreamEncoderTest,
4255 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004257 DataRate::BitsPerSec(kTargetBitrateBps),
4258 DataRate::BitsPerSec(kTargetBitrateBps),
4259 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004260 const int kWidth = 640;
4261 const int kHeight = 360;
4262
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004263 video_stream_encoder_->SetSource(&video_source_,
4264 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004265
4266 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4267 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004268 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004269 }
4270
mflodmancc3d4422017-08-03 08:27:51 -07004271 video_stream_encoder_->Stop();
4272 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004273 stats_proxy_.reset();
4274
4275 EXPECT_EQ(0,
4276 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4277}
4278
Per Kjellanderdcef6412020-10-07 15:09:05 +02004279TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4280 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004281 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004282 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004283
4284 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004285 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004286 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004287 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4288 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004289
Henrik Boström381d1092020-05-12 18:49:07 +02004290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004291 DataRate::BitsPerSec(kLowTargetBitrateBps),
4292 DataRate::BitsPerSec(kLowTargetBitrateBps),
4293 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004294
sprang57c2fff2017-01-16 06:24:02 -08004295 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004296 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4297 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004298 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4299 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4300
Erik Språngd7329ca2019-02-21 21:19:53 +01004301 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004302 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004303 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004304
Per Kjellanderdcef6412020-10-07 15:09:05 +02004305 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004306 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004307 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4308 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004309 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004310 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004311
Per Kjellanderdcef6412020-10-07 15:09:05 +02004312 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004313 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004314 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004315 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004316 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4317 WaitForEncodedFrame(CurrentTimeMs());
4318 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004319 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004320 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004321
mflodmancc3d4422017-08-03 08:27:51 -07004322 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004323}
4324
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004325TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004326 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004327 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004328 kVideoLayersAllocation);
4329
4330 const int kDefaultFps = 30;
4331
4332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4333 DataRate::BitsPerSec(kLowTargetBitrateBps),
4334 DataRate::BitsPerSec(kLowTargetBitrateBps),
4335 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4336
4337 video_source_.IncomingCapturedFrame(
4338 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4339 WaitForEncodedFrame(CurrentTimeMs());
4340 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4341 VideoLayersAllocation last_layer_allocation =
4342 sink_.GetLastVideoLayersAllocation();
4343 // kLowTargetBitrateBps is only enough for one spatial layer.
4344 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4345
4346 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004347 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004348 // Check that encoder has been updated too, not just allocation observer.
4349 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4350 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4351
Erik Språng9d69cbe2020-10-22 17:44:42 +02004352 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004353 int number_of_layers_allocation = 1;
4354 const int64_t start_time_ms = CurrentTimeMs();
4355 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4356 video_source_.IncomingCapturedFrame(
4357 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4358 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004359 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4360 number_of_layers_allocation = sink_.number_of_layers_allocations();
4361 VideoLayersAllocation new_allocation =
4362 sink_.GetLastVideoLayersAllocation();
4363 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4364 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4365 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4366 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4367 .target_bitrate_per_temporal_layer,
4368 last_layer_allocation.active_spatial_layers[0]
4369 .target_bitrate_per_temporal_layer);
4370 last_layer_allocation = new_allocation;
4371 }
4372 }
4373 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4374 video_stream_encoder_->Stop();
4375}
4376
4377TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004378 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004379 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4380 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4381 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004382 VideoEncoderConfig video_encoder_config;
4383 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4384 /* num_streams*/ 3, &video_encoder_config);
4385 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4386 video_encoder_config.content_type =
4387 VideoEncoderConfig::ContentType::kRealtimeVideo;
4388 video_encoder_config.encoder_specific_settings =
4389 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4390 VideoEncoder::GetDefaultVp8Settings());
4391 for (auto& layer : video_encoder_config.simulcast_layers) {
4392 layer.num_temporal_layers = 2;
4393 }
4394 // Simulcast layers are used for enabling/disabling streams.
4395 video_encoder_config.simulcast_layers[0].active = true;
4396 video_encoder_config.simulcast_layers[1].active = false;
4397 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004398 ConfigureEncoder(std::move(video_encoder_config),
4399 VideoStreamEncoder::BitrateAllocationCallbackType::
4400 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004401
4402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4403 DataRate::BitsPerSec(kTargetBitrateBps),
4404 DataRate::BitsPerSec(kTargetBitrateBps),
4405 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4406
4407 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4408 WaitForEncodedFrame(CurrentTimeMs());
4409 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4410 VideoLayersAllocation last_layer_allocation =
4411 sink_.GetLastVideoLayersAllocation();
4412
4413 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4414 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4415 .target_bitrate_per_temporal_layer,
4416 SizeIs(2));
4417 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4418 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4419 video_stream_encoder_->Stop();
4420}
4421
4422TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004423 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004424 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4425 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4426 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004427 VideoEncoderConfig video_encoder_config;
4428 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4429 /* num_streams*/ 3, &video_encoder_config);
4430 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4431 video_encoder_config.content_type =
4432 VideoEncoderConfig::ContentType::kRealtimeVideo;
4433 video_encoder_config.encoder_specific_settings =
4434 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4435 VideoEncoder::GetDefaultVp8Settings());
4436 for (auto& layer : video_encoder_config.simulcast_layers) {
4437 layer.num_temporal_layers = 2;
4438 }
4439 // Simulcast layers are used for enabling/disabling streams.
4440 video_encoder_config.simulcast_layers[0].active = true;
4441 video_encoder_config.simulcast_layers[1].active = false;
4442 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004443 ConfigureEncoder(std::move(video_encoder_config),
4444 VideoStreamEncoder::BitrateAllocationCallbackType::
4445 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004446
4447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4448 DataRate::BitsPerSec(kTargetBitrateBps),
4449 DataRate::BitsPerSec(kTargetBitrateBps),
4450 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4451
4452 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4453 WaitForEncodedFrame(CurrentTimeMs());
4454 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4455 VideoLayersAllocation last_layer_allocation =
4456 sink_.GetLastVideoLayersAllocation();
4457
4458 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4459 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4460 .target_bitrate_per_temporal_layer,
4461 SizeIs(2));
4462 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4463
4464 video_stream_encoder_->Stop();
4465}
4466
4467TEST_F(VideoStreamEncoderTest,
4468 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4469 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4470 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004471 VideoEncoderConfig video_encoder_config;
4472 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4473 /* num_streams*/ 1, &video_encoder_config);
4474 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4475 video_encoder_config.content_type =
4476 VideoEncoderConfig::ContentType::kRealtimeVideo;
4477 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4478 vp9_settings.numberOfSpatialLayers = 2;
4479 vp9_settings.numberOfTemporalLayers = 2;
4480 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4481 vp9_settings.automaticResizeOn = false;
4482 video_encoder_config.encoder_specific_settings =
4483 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4484 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004485 ConfigureEncoder(std::move(video_encoder_config),
4486 VideoStreamEncoder::BitrateAllocationCallbackType::
4487 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004488
4489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4490 DataRate::BitsPerSec(kTargetBitrateBps),
4491 DataRate::BitsPerSec(kTargetBitrateBps),
4492 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4493
4494 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4495 WaitForEncodedFrame(CurrentTimeMs());
4496 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4497 VideoLayersAllocation last_layer_allocation =
4498 sink_.GetLastVideoLayersAllocation();
4499
4500 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4501 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4502 .target_bitrate_per_temporal_layer,
4503 SizeIs(2));
4504 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4505 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4506 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4507 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4508 .target_bitrate_per_temporal_layer,
4509 SizeIs(2));
4510 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4511 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4512 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4513
4514 // Since full SVC is used, expect the top layer to utilize the full target
4515 // rate.
4516 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4517 .target_bitrate_per_temporal_layer[1],
4518 DataRate::BitsPerSec(kTargetBitrateBps));
4519 video_stream_encoder_->Stop();
4520}
4521
4522TEST_F(VideoStreamEncoderTest,
4523 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4524 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4525 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004526 VideoEncoderConfig video_encoder_config;
4527 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4528 /* num_streams*/ 1, &video_encoder_config);
4529 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4530 video_encoder_config.content_type =
4531 VideoEncoderConfig::ContentType::kRealtimeVideo;
4532 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4533 vp9_settings.numberOfSpatialLayers = 2;
4534 vp9_settings.numberOfTemporalLayers = 2;
4535 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4536 vp9_settings.automaticResizeOn = false;
4537 video_encoder_config.encoder_specific_settings =
4538 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4539 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004540 ConfigureEncoder(std::move(video_encoder_config),
4541 VideoStreamEncoder::BitrateAllocationCallbackType::
4542 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004543
4544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4545 DataRate::BitsPerSec(kTargetBitrateBps),
4546 DataRate::BitsPerSec(kTargetBitrateBps),
4547 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4548
4549 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4550 WaitForEncodedFrame(CurrentTimeMs());
4551 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4552 VideoLayersAllocation last_layer_allocation =
4553 sink_.GetLastVideoLayersAllocation();
4554
4555 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4556 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4557 .target_bitrate_per_temporal_layer,
4558 SizeIs(1));
4559 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4560 .target_bitrate_per_temporal_layer,
4561 SizeIs(1));
4562 // Since full SVC is used, expect the top layer to utilize the full target
4563 // rate.
4564 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4565 .target_bitrate_per_temporal_layer[0],
4566 DataRate::BitsPerSec(kTargetBitrateBps));
4567 video_stream_encoder_->Stop();
4568}
4569
4570TEST_F(VideoStreamEncoderTest,
4571 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4572 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4573 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004574 VideoEncoderConfig video_encoder_config;
4575 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4576 /* num_streams*/ 1, &video_encoder_config);
4577 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4578 video_encoder_config.content_type =
4579 VideoEncoderConfig::ContentType::kRealtimeVideo;
4580 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4581 vp9_settings.numberOfSpatialLayers = 2;
4582 vp9_settings.numberOfTemporalLayers = 2;
4583 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4584 vp9_settings.automaticResizeOn = false;
4585 video_encoder_config.encoder_specific_settings =
4586 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4587 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004588 ConfigureEncoder(std::move(video_encoder_config),
4589 VideoStreamEncoder::BitrateAllocationCallbackType::
4590 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004591
4592 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4593 DataRate::BitsPerSec(kTargetBitrateBps),
4594 DataRate::BitsPerSec(kTargetBitrateBps),
4595 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4596
4597 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4598 WaitForEncodedFrame(CurrentTimeMs());
4599 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4600 VideoLayersAllocation last_layer_allocation =
4601 sink_.GetLastVideoLayersAllocation();
4602
4603 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4604 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4605 .target_bitrate_per_temporal_layer,
4606 SizeIs(2));
4607 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4608 .target_bitrate_per_temporal_layer,
4609 SizeIs(2));
4610 // Since KSVC is, spatial layers are independend except on key frames.
4611 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4612 .target_bitrate_per_temporal_layer[1],
4613 DataRate::BitsPerSec(kTargetBitrateBps));
4614 video_stream_encoder_->Stop();
4615}
4616
4617TEST_F(VideoStreamEncoderTest,
4618 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4619 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4620 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4621 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004622 VideoEncoderConfig video_encoder_config;
4623 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4624 /* num_streams*/ 1, &video_encoder_config);
4625 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4626 video_encoder_config.content_type =
4627 VideoEncoderConfig::ContentType::kRealtimeVideo;
4628 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4629 vp9_settings.numberOfSpatialLayers = 3;
4630 vp9_settings.numberOfTemporalLayers = 2;
4631 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4632 vp9_settings.automaticResizeOn = false;
4633 video_encoder_config.encoder_specific_settings =
4634 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4635 vp9_settings);
4636 // Simulcast layers are used for enabling/disabling streams.
4637 video_encoder_config.simulcast_layers.resize(3);
4638 video_encoder_config.simulcast_layers[0].active = false;
4639 video_encoder_config.simulcast_layers[1].active = true;
4640 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004641 ConfigureEncoder(std::move(video_encoder_config),
4642 VideoStreamEncoder::BitrateAllocationCallbackType::
4643 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004644
4645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4646 DataRate::BitsPerSec(kTargetBitrateBps),
4647 DataRate::BitsPerSec(kTargetBitrateBps),
4648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4649
4650 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4651 WaitForEncodedFrame(CurrentTimeMs());
4652 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4653 VideoLayersAllocation last_layer_allocation =
4654 sink_.GetLastVideoLayersAllocation();
4655
4656 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4657 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4658 .target_bitrate_per_temporal_layer,
4659 SizeIs(2));
4660 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4661 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4662
4663 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4664 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4665 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4666 .target_bitrate_per_temporal_layer,
4667 SizeIs(2));
4668 // Since full SVC is used, expect the top layer to utilize the full target
4669 // rate.
4670 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4671 .target_bitrate_per_temporal_layer[1],
4672 DataRate::BitsPerSec(kTargetBitrateBps));
4673 video_stream_encoder_->Stop();
4674}
4675
4676TEST_F(VideoStreamEncoderTest,
4677 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4678 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4679 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4680 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004681 VideoEncoderConfig video_encoder_config;
4682 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4683 /* num_streams*/ 1, &video_encoder_config);
4684 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4685 video_encoder_config.content_type =
4686 VideoEncoderConfig::ContentType::kRealtimeVideo;
4687 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4688 vp9_settings.numberOfSpatialLayers = 3;
4689 vp9_settings.numberOfTemporalLayers = 2;
4690 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4691 vp9_settings.automaticResizeOn = false;
4692 video_encoder_config.encoder_specific_settings =
4693 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4694 vp9_settings);
4695 // Simulcast layers are used for enabling/disabling streams.
4696 video_encoder_config.simulcast_layers.resize(3);
4697 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004698 ConfigureEncoder(std::move(video_encoder_config),
4699 VideoStreamEncoder::BitrateAllocationCallbackType::
4700 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004701
4702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4703 DataRate::BitsPerSec(kTargetBitrateBps),
4704 DataRate::BitsPerSec(kTargetBitrateBps),
4705 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4706
4707 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4708 WaitForEncodedFrame(CurrentTimeMs());
4709 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4710 VideoLayersAllocation last_layer_allocation =
4711 sink_.GetLastVideoLayersAllocation();
4712
4713 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4714 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4715 .target_bitrate_per_temporal_layer,
4716 SizeIs(2));
4717 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4718 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4719
4720 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4721 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4722 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4723 .target_bitrate_per_temporal_layer,
4724 SizeIs(2));
4725 video_stream_encoder_->Stop();
4726}
4727
4728TEST_F(VideoStreamEncoderTest,
4729 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4730 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4731 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4732 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004733 VideoEncoderConfig video_encoder_config;
4734 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4735 /* num_streams*/ 1, &video_encoder_config);
4736 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4737 video_encoder_config.content_type =
4738 VideoEncoderConfig::ContentType::kRealtimeVideo;
4739 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4740 vp9_settings.numberOfSpatialLayers = 3;
4741 vp9_settings.numberOfTemporalLayers = 2;
4742 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4743 vp9_settings.automaticResizeOn = false;
4744 video_encoder_config.encoder_specific_settings =
4745 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4746 vp9_settings);
4747 // Simulcast layers are used for enabling/disabling streams.
4748 video_encoder_config.simulcast_layers.resize(3);
4749 video_encoder_config.simulcast_layers[0].active = false;
4750 video_encoder_config.simulcast_layers[1].active = false;
4751 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004752 ConfigureEncoder(std::move(video_encoder_config),
4753 VideoStreamEncoder::BitrateAllocationCallbackType::
4754 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755
4756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4757 DataRate::BitsPerSec(kTargetBitrateBps),
4758 DataRate::BitsPerSec(kTargetBitrateBps),
4759 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4760
4761 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4762 WaitForEncodedFrame(CurrentTimeMs());
4763 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4764 VideoLayersAllocation last_layer_allocation =
4765 sink_.GetLastVideoLayersAllocation();
4766
4767 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4768 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4769 .target_bitrate_per_temporal_layer,
4770 SizeIs(2));
4771 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4772 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4773 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4774 .target_bitrate_per_temporal_layer[1],
4775 DataRate::BitsPerSec(kTargetBitrateBps));
4776 video_stream_encoder_->Stop();
4777}
4778
4779TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4780 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004781 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004782 kVideoLayersAllocation);
4783 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4784 DataRate::BitsPerSec(kTargetBitrateBps),
4785 DataRate::BitsPerSec(kTargetBitrateBps),
4786 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4787
4788 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4789 WaitForEncodedFrame(CurrentTimeMs());
4790 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4791 VideoLayersAllocation last_layer_allocation =
4792 sink_.GetLastVideoLayersAllocation();
4793
4794 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4795 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4796 .target_bitrate_per_temporal_layer,
4797 SizeIs(1));
4798 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4799 .target_bitrate_per_temporal_layer[0],
4800 DataRate::BitsPerSec(kTargetBitrateBps));
4801 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4802 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4803 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4804 video_stream_encoder_->Stop();
4805}
4806
4807TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004808 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4809 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004810 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004811 kVideoLayersAllocation);
4812
4813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4814 DataRate::BitsPerSec(kLowTargetBitrateBps),
4815 DataRate::BitsPerSec(kLowTargetBitrateBps),
4816 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4817
4818 video_source_.IncomingCapturedFrame(
4819 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4820 WaitForEncodedFrame(CurrentTimeMs());
4821 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4822 VideoLayersAllocation last_layer_allocation =
4823 sink_.GetLastVideoLayersAllocation();
4824 // kLowTargetBitrateBps is only enough for one spatial layer.
4825 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4826 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4827 .target_bitrate_per_temporal_layer[0],
4828 DataRate::BitsPerSec(kLowTargetBitrateBps));
4829
4830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4831 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4832 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4833 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4834 video_source_.IncomingCapturedFrame(
4835 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4836 WaitForEncodedFrame(CurrentTimeMs());
4837
4838 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4839 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4840 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4841 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4842 .target_bitrate_per_temporal_layer[0],
4843 DataRate::Zero());
4844
4845 video_stream_encoder_->Stop();
4846}
4847
Per Kjellander4190ce92020-12-15 17:24:55 +01004848TEST_F(VideoStreamEncoderTest,
4849 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4850 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004851 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004852 kVideoLayersAllocation);
4853
4854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4855 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4856 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4857 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4858
4859 video_source_.IncomingCapturedFrame(
4860 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4861 WaitForEncodedFrame(CurrentTimeMs());
4862 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4863 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4864 SizeIs(2));
4865 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4866 codec_width_);
4867 EXPECT_EQ(
4868 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4869 codec_height_);
4870
4871 video_source_.IncomingCapturedFrame(
4872 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4873 WaitForEncodedFrame(CurrentTimeMs());
4874 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4875 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4876 SizeIs(2));
4877 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4878 codec_width_ / 2);
4879 EXPECT_EQ(
4880 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4881 codec_height_ / 2);
4882
4883 video_stream_encoder_->Stop();
4884}
4885
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004886TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4887 // 2 TLs configured, temporal layers supported by encoder.
4888 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004889 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004890 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004891 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004892 fake_encoder_.SetTemporalLayersSupported(0, true);
4893
4894 // Bitrate allocated across temporal layers.
4895 const int kTl0Bps = kTargetBitrateBps *
4896 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004897 kNumTemporalLayers, /*temporal_id*/ 0,
4898 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004899 const int kTl1Bps = kTargetBitrateBps *
4900 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004901 kNumTemporalLayers, /*temporal_id*/ 1,
4902 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004903 VideoBitrateAllocation expected_bitrate;
4904 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4905 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4906
4907 VerifyAllocatedBitrate(expected_bitrate);
4908 video_stream_encoder_->Stop();
4909}
4910
4911TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4912 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004913 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004914 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004915 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004916 fake_encoder_.SetTemporalLayersSupported(0, false);
4917
4918 // Temporal layers not supported by the encoder.
4919 // Total bitrate should be at ti:0.
4920 VideoBitrateAllocation expected_bitrate;
4921 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4922
4923 VerifyAllocatedBitrate(expected_bitrate);
4924 video_stream_encoder_->Stop();
4925}
4926
4927TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004928 webrtc::test::ScopedFieldTrials field_trials(
4929 "WebRTC-Video-QualityScalerSettings/"
4930 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4931 // Reset encoder for field trials to take effect.
4932 ConfigureEncoder(video_encoder_config_.Copy());
4933
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004934 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004935 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004936 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004937 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004938 fake_encoder_.SetTemporalLayersSupported(0, true);
4939 fake_encoder_.SetTemporalLayersSupported(1, false);
4940
4941 const int kS0Bps = 150000;
4942 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004943 kS0Bps *
4944 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4945 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004946 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004947 kS0Bps *
4948 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4949 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004950 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4951 // Temporal layers not supported by si:1.
4952 VideoBitrateAllocation expected_bitrate;
4953 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4954 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4955 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4956
4957 VerifyAllocatedBitrate(expected_bitrate);
4958 video_stream_encoder_->Stop();
4959}
4960
Niels Möller7dc26b72017-12-06 10:27:48 +01004961TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4962 const int kFrameWidth = 1280;
4963 const int kFrameHeight = 720;
4964 const int kFramerate = 24;
4965
Henrik Boström381d1092020-05-12 18:49:07 +02004966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004967 DataRate::BitsPerSec(kTargetBitrateBps),
4968 DataRate::BitsPerSec(kTargetBitrateBps),
4969 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004970 test::FrameForwarder source;
4971 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004972 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004973
4974 // Insert a single frame, triggering initial configuration.
4975 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4977
4978 EXPECT_EQ(
4979 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4980 kDefaultFramerate);
4981
4982 // Trigger reconfigure encoder (without resetting the entire instance).
4983 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004984 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4985 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004986 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004987 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004988 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004989 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4990
4991 // Detector should be updated with fps limit from codec config.
4992 EXPECT_EQ(
4993 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4994 kFramerate);
4995
4996 // Trigger overuse, max framerate should be reduced.
4997 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4998 stats.input_frame_rate = kFramerate;
4999 stats_proxy_->SetMockStats(stats);
5000 video_stream_encoder_->TriggerCpuOveruse();
5001 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5002 int adapted_framerate =
5003 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5004 EXPECT_LT(adapted_framerate, kFramerate);
5005
5006 // Trigger underuse, max framerate should go back to codec configured fps.
5007 // Set extra low fps, to make sure it's actually reset, not just incremented.
5008 stats = stats_proxy_->GetStats();
5009 stats.input_frame_rate = adapted_framerate / 2;
5010 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005011 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005012 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5013 EXPECT_EQ(
5014 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5015 kFramerate);
5016
5017 video_stream_encoder_->Stop();
5018}
5019
5020TEST_F(VideoStreamEncoderTest,
5021 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5022 const int kFrameWidth = 1280;
5023 const int kFrameHeight = 720;
5024 const int kLowFramerate = 15;
5025 const int kHighFramerate = 25;
5026
Henrik Boström381d1092020-05-12 18:49:07 +02005027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005028 DataRate::BitsPerSec(kTargetBitrateBps),
5029 DataRate::BitsPerSec(kTargetBitrateBps),
5030 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005031 test::FrameForwarder source;
5032 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005033 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005034
5035 // Trigger initial configuration.
5036 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005037 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5038 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005039 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005040 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005041 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005042 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005043 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5044
5045 EXPECT_EQ(
5046 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5047 kLowFramerate);
5048
5049 // Trigger overuse, max framerate should be reduced.
5050 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5051 stats.input_frame_rate = kLowFramerate;
5052 stats_proxy_->SetMockStats(stats);
5053 video_stream_encoder_->TriggerCpuOveruse();
5054 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5055 int adapted_framerate =
5056 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5057 EXPECT_LT(adapted_framerate, kLowFramerate);
5058
5059 // Reconfigure the encoder with a new (higher max framerate), max fps should
5060 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005061 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005062 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5063 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005064 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005065 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5066
5067 EXPECT_EQ(
5068 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5069 adapted_framerate);
5070
5071 // Trigger underuse, max framerate should go back to codec configured fps.
5072 stats = stats_proxy_->GetStats();
5073 stats.input_frame_rate = adapted_framerate;
5074 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005075 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005076 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5077 EXPECT_EQ(
5078 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5079 kHighFramerate);
5080
5081 video_stream_encoder_->Stop();
5082}
5083
mflodmancc3d4422017-08-03 08:27:51 -07005084TEST_F(VideoStreamEncoderTest,
5085 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005086 const int kFrameWidth = 1280;
5087 const int kFrameHeight = 720;
5088 const int kFramerate = 24;
5089
Henrik Boström381d1092020-05-12 18:49:07 +02005090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005091 DataRate::BitsPerSec(kTargetBitrateBps),
5092 DataRate::BitsPerSec(kTargetBitrateBps),
5093 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005094 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005096 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005097
5098 // Trigger initial configuration.
5099 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005100 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5101 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005102 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005103 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005104 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005105 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005106 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005107
Niels Möller7dc26b72017-12-06 10:27:48 +01005108 EXPECT_EQ(
5109 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5110 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005111
5112 // Trigger overuse, max framerate should be reduced.
5113 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5114 stats.input_frame_rate = kFramerate;
5115 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005116 video_stream_encoder_->TriggerCpuOveruse();
5117 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005118 int adapted_framerate =
5119 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005120 EXPECT_LT(adapted_framerate, kFramerate);
5121
5122 // Change degradation preference to not enable framerate scaling. Target
5123 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005124 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005125 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005126 EXPECT_EQ(
5127 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5128 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005129
mflodmancc3d4422017-08-03 08:27:51 -07005130 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005131}
5132
mflodmancc3d4422017-08-03 08:27:51 -07005133TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005134 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005136 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5137 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5138 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005139 const int kWidth = 640;
5140 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005141
asaperssonfab67072017-04-04 05:51:49 -07005142 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005143
5144 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005145 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005146
5147 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005148 EXPECT_TRUE_WAIT(
5149 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005150
sprangc5d62e22017-04-02 23:53:04 -07005151 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005152
asaperssonfab67072017-04-04 05:51:49 -07005153 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005154 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005155 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005156
5157 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005158 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005159
Henrik Boström2671dac2020-05-19 16:29:09 +02005160 EXPECT_TRUE_WAIT(
5161 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005162
mflodmancc3d4422017-08-03 08:27:51 -07005163 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005164}
5165
mflodmancc3d4422017-08-03 08:27:51 -07005166TEST_F(VideoStreamEncoderTest,
5167 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005168 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005170 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5171 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5172 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005173 const int kWidth = 640;
5174 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005175
5176 // We expect the n initial frames to get dropped.
5177 int i;
5178 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005179 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005180 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005181 }
5182 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005183 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005184 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005185
5186 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005187 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005188
mflodmancc3d4422017-08-03 08:27:51 -07005189 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005190}
5191
mflodmancc3d4422017-08-03 08:27:51 -07005192TEST_F(VideoStreamEncoderTest,
5193 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005194 const int kWidth = 640;
5195 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005196 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005197 DataRate::BitsPerSec(kLowTargetBitrateBps),
5198 DataRate::BitsPerSec(kLowTargetBitrateBps),
5199 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005200
5201 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005202 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005203 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005204
asaperssonfab67072017-04-04 05:51:49 -07005205 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005206 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005207 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005208
mflodmancc3d4422017-08-03 08:27:51 -07005209 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005210}
5211
mflodmancc3d4422017-08-03 08:27:51 -07005212TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005213 const int kWidth = 640;
5214 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005215 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005216
5217 VideoEncoderConfig video_encoder_config;
5218 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5219 // Make format different, to force recreation of encoder.
5220 video_encoder_config.video_format.parameters["foo"] = "foo";
5221 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005222 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005224 DataRate::BitsPerSec(kLowTargetBitrateBps),
5225 DataRate::BitsPerSec(kLowTargetBitrateBps),
5226 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005227
kthelgasonb83797b2017-02-14 11:57:25 -08005228 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005229 video_stream_encoder_->SetSource(&video_source_,
5230 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005231
asaperssonfab67072017-04-04 05:51:49 -07005232 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005233 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005234 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005235
mflodmancc3d4422017-08-03 08:27:51 -07005236 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005237 fake_encoder_.SetQualityScaling(true);
5238}
5239
Åsa Persson139f4dc2019-08-02 09:29:58 +02005240TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5241 webrtc::test::ScopedFieldTrials field_trials(
5242 "WebRTC-Video-QualityScalerSettings/"
5243 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5244 // Reset encoder for field trials to take effect.
5245 ConfigureEncoder(video_encoder_config_.Copy());
5246 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5247 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5248 const int kWidth = 640;
5249 const int kHeight = 360;
5250
Henrik Boström381d1092020-05-12 18:49:07 +02005251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005252 DataRate::BitsPerSec(kTargetBitrateBps),
5253 DataRate::BitsPerSec(kTargetBitrateBps),
5254 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005255 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5256 // Frame should not be dropped.
5257 WaitForEncodedFrame(1);
5258
Henrik Boström381d1092020-05-12 18:49:07 +02005259 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005260 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5261 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5262 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005263 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5264 // Frame should not be dropped.
5265 WaitForEncodedFrame(2);
5266
Henrik Boström381d1092020-05-12 18:49:07 +02005267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005268 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5269 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5270 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005271 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5272 // Expect to drop this frame, the wait should time out.
5273 ExpectDroppedFrame();
5274
5275 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005276 EXPECT_TRUE_WAIT(
5277 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005278 video_stream_encoder_->Stop();
5279}
5280
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005281TEST_F(VideoStreamEncoderTest,
5282 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5283 webrtc::test::ScopedFieldTrials field_trials(
5284 "WebRTC-Video-QualityScalerSettings/"
5285 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5286 fake_encoder_.SetQualityScaling(false);
5287 ConfigureEncoder(video_encoder_config_.Copy());
5288 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5289 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5290 const int kWidth = 640;
5291 const int kHeight = 360;
5292
5293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5294 DataRate::BitsPerSec(kTargetBitrateBps),
5295 DataRate::BitsPerSec(kTargetBitrateBps),
5296 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5297 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5298 // Frame should not be dropped.
5299 WaitForEncodedFrame(1);
5300
5301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5302 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5303 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5304 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5305 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5306 // Frame should not be dropped.
5307 WaitForEncodedFrame(2);
5308
5309 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5310 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5311 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5312 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5313 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5314 // Not dropped since quality scaling is disabled.
5315 WaitForEncodedFrame(3);
5316
5317 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005318 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005319 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5320
5321 video_stream_encoder_->Stop();
5322}
5323
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005324TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5325 const int kLowTargetBitrateBps = 400000;
5326 // Set simulcast.
5327 ResetEncoder("VP8", 3, 1, 1, false);
5328 fake_encoder_.SetQualityScaling(true);
5329 const int kWidth = 1280;
5330 const int kHeight = 720;
5331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5332 DataRate::BitsPerSec(kLowTargetBitrateBps),
5333 DataRate::BitsPerSec(kLowTargetBitrateBps),
5334 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5335 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5336 // Frame should not be dropped.
5337 WaitForEncodedFrame(1);
5338
5339 // Trigger QVGA "singlecast"
5340 // Update the config.
5341 VideoEncoderConfig video_encoder_config;
5342 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5343 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005344 video_encoder_config.video_stream_factory =
5345 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5346 "VP8", /*max qp*/ 56, /*screencast*/ false,
5347 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005348 for (auto& layer : video_encoder_config.simulcast_layers) {
5349 layer.num_temporal_layers = 1;
5350 layer.max_framerate = kDefaultFramerate;
5351 }
5352 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5353 video_encoder_config.content_type =
5354 VideoEncoderConfig::ContentType::kRealtimeVideo;
5355
5356 video_encoder_config.simulcast_layers[0].active = true;
5357 video_encoder_config.simulcast_layers[1].active = false;
5358 video_encoder_config.simulcast_layers[2].active = false;
5359
5360 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5361 kMaxPayloadLength);
5362 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5363
5364 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5365 // Frame should not be dropped.
5366 WaitForEncodedFrame(2);
5367
5368 // Trigger HD "singlecast"
5369 video_encoder_config.simulcast_layers[0].active = false;
5370 video_encoder_config.simulcast_layers[1].active = false;
5371 video_encoder_config.simulcast_layers[2].active = true;
5372
5373 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5374 kMaxPayloadLength);
5375 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5376
5377 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5378 // Frame should be dropped because of initial frame drop.
5379 ExpectDroppedFrame();
5380
5381 // Expect the sink_wants to specify a scaled frame.
5382 EXPECT_TRUE_WAIT(
5383 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5384 video_stream_encoder_->Stop();
5385}
5386
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005387TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5388 const int kLowTargetBitrateBps = 400000;
5389 // Set simulcast.
5390 ResetEncoder("VP9", 1, 1, 3, false);
5391 fake_encoder_.SetQualityScaling(true);
5392 const int kWidth = 1280;
5393 const int kHeight = 720;
5394 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5395 DataRate::BitsPerSec(kLowTargetBitrateBps),
5396 DataRate::BitsPerSec(kLowTargetBitrateBps),
5397 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5398 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5399 // Frame should not be dropped.
5400 WaitForEncodedFrame(1);
5401
5402 // Trigger QVGA "singlecast"
5403 // Update the config.
5404 VideoEncoderConfig video_encoder_config;
5405 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5406 &video_encoder_config);
5407 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5408 vp9_settings.numberOfSpatialLayers = 3;
5409 // Since only one layer is active - automatic resize should be enabled.
5410 vp9_settings.automaticResizeOn = true;
5411 video_encoder_config.encoder_specific_settings =
5412 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5413 vp9_settings);
5414 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5415 video_encoder_config.content_type =
5416 VideoEncoderConfig::ContentType::kRealtimeVideo;
5417 // Currently simulcast layers |active| flags are used to inidicate
5418 // which SVC layers are active.
5419 video_encoder_config.simulcast_layers.resize(3);
5420
5421 video_encoder_config.simulcast_layers[0].active = true;
5422 video_encoder_config.simulcast_layers[1].active = false;
5423 video_encoder_config.simulcast_layers[2].active = false;
5424
5425 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5426 kMaxPayloadLength);
5427 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5428
5429 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5430 // Frame should not be dropped.
5431 WaitForEncodedFrame(2);
5432
5433 // Trigger HD "singlecast"
5434 video_encoder_config.simulcast_layers[0].active = false;
5435 video_encoder_config.simulcast_layers[1].active = false;
5436 video_encoder_config.simulcast_layers[2].active = true;
5437
5438 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5439 kMaxPayloadLength);
5440 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5441
5442 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5443 // Frame should be dropped because of initial frame drop.
5444 ExpectDroppedFrame();
5445
5446 // Expect the sink_wants to specify a scaled frame.
5447 EXPECT_TRUE_WAIT(
5448 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5449 video_stream_encoder_->Stop();
5450}
5451
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005452TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005453 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5454 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5455 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5456 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5457 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5458 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5459 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5460 fake_encoder_.SetResolutionBitrateLimits(
5461 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5462
5463 VideoEncoderConfig video_encoder_config;
5464 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5465 &video_encoder_config);
5466 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5467 vp9_settings.numberOfSpatialLayers = 3;
5468 // Since only one layer is active - automatic resize should be enabled.
5469 vp9_settings.automaticResizeOn = true;
5470 video_encoder_config.encoder_specific_settings =
5471 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5472 vp9_settings);
5473 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5474 video_encoder_config.content_type =
5475 VideoEncoderConfig::ContentType::kRealtimeVideo;
5476 // Simulcast layers are used to indicate which spatial layers are active.
5477 video_encoder_config.simulcast_layers.resize(3);
5478 video_encoder_config.simulcast_layers[0].active = false;
5479 video_encoder_config.simulcast_layers[1].active = true;
5480 video_encoder_config.simulcast_layers[2].active = false;
5481
5482 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5483 kMaxPayloadLength);
5484 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5485
5486 // The encoder bitrate limits for 360p should be used.
5487 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5488 EXPECT_FALSE(WaitForFrame(1000));
5489 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5490 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5491 VideoCodecType::kVideoCodecVP9);
5492 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5493 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5494 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5495 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5496 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5497 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5498 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5499 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5500
5501 // The encoder bitrate limits for 270p should be used.
5502 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5503 EXPECT_FALSE(WaitForFrame(1000));
5504 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5505 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5506 VideoCodecType::kVideoCodecVP9);
5507 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5508 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5509 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5510 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5511 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5512 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5513 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5514 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5515
5516 video_stream_encoder_->Stop();
5517}
5518
5519TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005520 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5521 VideoEncoderConfig video_encoder_config;
5522 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5523 &video_encoder_config);
5524 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5525 vp9_settings.numberOfSpatialLayers = 3;
5526 // Since only one layer is active - automatic resize should be enabled.
5527 vp9_settings.automaticResizeOn = true;
5528 video_encoder_config.encoder_specific_settings =
5529 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5530 vp9_settings);
5531 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5532 video_encoder_config.content_type =
5533 VideoEncoderConfig::ContentType::kRealtimeVideo;
5534 // Simulcast layers are used to indicate which spatial layers are active.
5535 video_encoder_config.simulcast_layers.resize(3);
5536 video_encoder_config.simulcast_layers[0].active = false;
5537 video_encoder_config.simulcast_layers[1].active = true;
5538 video_encoder_config.simulcast_layers[2].active = false;
5539
5540 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5541 kMaxPayloadLength);
5542 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5543
5544 // The default bitrate limits for 360p should be used.
5545 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005546 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5547 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005548 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5549 EXPECT_FALSE(WaitForFrame(1000));
5550 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5551 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5552 VideoCodecType::kVideoCodecVP9);
5553 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5554 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5555 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5556 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5557 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5558 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5559 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5560 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5561
5562 // The default bitrate limits for 270p should be used.
5563 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005564 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5565 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005566 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5567 EXPECT_FALSE(WaitForFrame(1000));
5568 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5569 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5570 VideoCodecType::kVideoCodecVP9);
5571 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5572 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5573 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5574 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5575 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5576 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5577 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5578 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5579
5580 video_stream_encoder_->Stop();
5581}
5582
5583TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5584 webrtc::test::ScopedFieldTrials field_trials(
5585 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5586 VideoEncoderConfig video_encoder_config;
5587 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5588 &video_encoder_config);
5589 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5590 vp9_settings.numberOfSpatialLayers = 3;
5591 // Since only one layer is active - automatic resize should be enabled.
5592 vp9_settings.automaticResizeOn = true;
5593 video_encoder_config.encoder_specific_settings =
5594 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5595 vp9_settings);
5596 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5597 video_encoder_config.content_type =
5598 VideoEncoderConfig::ContentType::kRealtimeVideo;
5599 // Simulcast layers are used to indicate which spatial layers are active.
5600 video_encoder_config.simulcast_layers.resize(3);
5601 video_encoder_config.simulcast_layers[0].active = false;
5602 video_encoder_config.simulcast_layers[1].active = true;
5603 video_encoder_config.simulcast_layers[2].active = false;
5604
5605 // Reset encoder for field trials to take effect.
5606 ConfigureEncoder(video_encoder_config.Copy());
5607
5608 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5609 kMaxPayloadLength);
5610 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5611
5612 // The default bitrate limits for 360p should not be used.
5613 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005614 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5615 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005616 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5617 EXPECT_FALSE(WaitForFrame(1000));
5618 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005619 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005620 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5621 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5622 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5623 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5624 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5625 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5626
5627 video_stream_encoder_->Stop();
5628}
5629
5630TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5631 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5632 /*num_spatial_layers=*/1, /*screenshare=*/false);
5633
5634 // The default singlecast bitrate limits for 720p should not be used.
5635 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005636 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5637 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005638 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5639 EXPECT_FALSE(WaitForFrame(1000));
5640 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5641 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5642 VideoCodecType::kVideoCodecVP9);
5643 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5644 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5645 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5646 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5647 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5648 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5649
5650 video_stream_encoder_->Stop();
5651}
5652
5653TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005654 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5655 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5656 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5657 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5658 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5659 fake_encoder_.SetResolutionBitrateLimits(
5660 {kEncoderLimits180p, kEncoderLimits720p});
5661
5662 VideoEncoderConfig video_encoder_config;
5663 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5664 &video_encoder_config);
5665 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5666 vp9_settings.numberOfSpatialLayers = 3;
5667 // Since only one layer is active - automatic resize should be enabled.
5668 vp9_settings.automaticResizeOn = true;
5669 video_encoder_config.encoder_specific_settings =
5670 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5671 vp9_settings);
5672 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5673 video_encoder_config.content_type =
5674 VideoEncoderConfig::ContentType::kRealtimeVideo;
5675 // Simulcast layers are used to indicate which spatial layers are active.
5676 video_encoder_config.simulcast_layers.resize(3);
5677 video_encoder_config.simulcast_layers[0].active = true;
5678 video_encoder_config.simulcast_layers[1].active = false;
5679 video_encoder_config.simulcast_layers[2].active = false;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 // Limits not applied on lowest stream, limits for 180p should not be used.
5686 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5687 EXPECT_FALSE(WaitForFrame(1000));
5688 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5689 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5690 VideoCodecType::kVideoCodecVP9);
5691 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5692 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5693 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5694 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5695 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5696 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5697 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5698 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5699
5700 video_stream_encoder_->Stop();
5701}
5702
5703TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005704 InitialFrameDropActivatesWhenResolutionIncreases) {
5705 const int kWidth = 640;
5706 const int kHeight = 360;
5707
5708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5709 DataRate::BitsPerSec(kTargetBitrateBps),
5710 DataRate::BitsPerSec(kTargetBitrateBps),
5711 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5712 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5713 // Frame should not be dropped.
5714 WaitForEncodedFrame(1);
5715
5716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5717 DataRate::BitsPerSec(kLowTargetBitrateBps),
5718 DataRate::BitsPerSec(kLowTargetBitrateBps),
5719 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5720 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5721 // Frame should not be dropped, bitrate not too low for frame.
5722 WaitForEncodedFrame(2);
5723
5724 // Incoming resolution increases.
5725 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5726 // Expect to drop this frame, bitrate too low for frame.
5727 ExpectDroppedFrame();
5728
5729 // Expect the sink_wants to specify a scaled frame.
5730 EXPECT_TRUE_WAIT(
5731 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5732 video_stream_encoder_->Stop();
5733}
5734
5735TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5736 const int kWidth = 640;
5737 const int kHeight = 360;
5738 // So that quality scaling doesn't happen by itself.
5739 fake_encoder_.SetQp(kQpHigh);
5740
5741 AdaptingFrameForwarder source(&time_controller_);
5742 source.set_adaptation_enabled(true);
5743 video_stream_encoder_->SetSource(
5744 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5745
5746 int timestamp = 1;
5747
5748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5749 DataRate::BitsPerSec(kTargetBitrateBps),
5750 DataRate::BitsPerSec(kTargetBitrateBps),
5751 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5752 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5753 WaitForEncodedFrame(timestamp);
5754 timestamp += 9000;
5755 // Long pause to disable all first BWE drop logic.
5756 AdvanceTime(TimeDelta::Millis(1000));
5757
5758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5759 DataRate::BitsPerSec(kLowTargetBitrateBps),
5760 DataRate::BitsPerSec(kLowTargetBitrateBps),
5761 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5762 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5763 // Not dropped frame, as initial frame drop is disabled by now.
5764 WaitForEncodedFrame(timestamp);
5765 timestamp += 9000;
5766 AdvanceTime(TimeDelta::Millis(100));
5767
5768 // Quality adaptation down.
5769 video_stream_encoder_->TriggerQualityLow();
5770
5771 // Adaptation has an effect.
5772 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5773 5000);
5774
5775 // Frame isn't dropped as initial frame dropper is disabled.
5776 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5777 WaitForEncodedFrame(timestamp);
5778 timestamp += 9000;
5779 AdvanceTime(TimeDelta::Millis(100));
5780
5781 // Quality adaptation up.
5782 video_stream_encoder_->TriggerQualityHigh();
5783
5784 // Adaptation has an effect.
5785 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5786 5000);
5787
5788 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5789 // Frame should not be dropped, as initial framedropper is off.
5790 WaitForEncodedFrame(timestamp);
5791
5792 video_stream_encoder_->Stop();
5793}
5794
Åsa Persson7f354f82021-02-04 15:52:15 +01005795TEST_F(VideoStreamEncoderTest,
5796 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5797 const int kMinStartBps360p = 222000;
5798 fake_encoder_.SetResolutionBitrateLimits(
5799 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5800 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5801 800000)});
5802
5803 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5804 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5805 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5806 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5807 0, 0, 0);
5808 // Frame should not be dropped, bitrate not too low for frame.
5809 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5810 WaitForEncodedFrame(1);
5811
5812 // Incoming resolution increases, initial frame drop activates.
5813 // Frame should be dropped, link allocation too low for frame.
5814 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5815 ExpectDroppedFrame();
5816
5817 // Expect sink_wants to specify a scaled frame.
5818 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5819 5000);
5820 video_stream_encoder_->Stop();
5821}
5822
5823TEST_F(VideoStreamEncoderTest,
5824 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5825 const int kMinStartBps360p = 222000;
5826 fake_encoder_.SetResolutionBitrateLimits(
5827 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5828 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5829 800000)});
5830
5831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5832 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5833 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5834 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5835 0, 0, 0);
5836 // Frame should not be dropped, bitrate not too low for frame.
5837 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5838 WaitForEncodedFrame(1);
5839
5840 // Incoming resolution increases, initial frame drop activates.
5841 // Frame should be dropped, link allocation not too low for frame.
5842 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5843 WaitForEncodedFrame(2);
5844
5845 video_stream_encoder_->Stop();
5846}
5847
Åsa Perssone644a032019-11-08 15:56:00 +01005848TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5849 webrtc::test::ScopedFieldTrials field_trials(
5850 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5851
5852 // Reset encoder for field trials to take effect.
5853 VideoEncoderConfig config = video_encoder_config_.Copy();
5854 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005855 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005856 ConfigureEncoder(std::move(config));
5857 fake_encoder_.SetQp(kQpLow);
5858
5859 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005860 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005861 source.set_adaptation_enabled(true);
5862 video_stream_encoder_->SetSource(&source,
5863 DegradationPreference::MAINTAIN_FRAMERATE);
5864
5865 // Start at low bitrate.
5866 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005867 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5868 DataRate::BitsPerSec(kLowBitrateBps),
5869 DataRate::BitsPerSec(kLowBitrateBps),
5870 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005871
5872 // Expect first frame to be dropped and resolution to be limited.
5873 const int kWidth = 1280;
5874 const int kHeight = 720;
5875 const int64_t kFrameIntervalMs = 100;
5876 int64_t timestamp_ms = kFrameIntervalMs;
5877 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5878 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005879 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5880 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005881
5882 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005883 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5884 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005885
5886 // Insert frames and advance |min_duration_ms|.
5887 for (size_t i = 1; i <= 10; i++) {
5888 timestamp_ms += kFrameIntervalMs;
5889 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5890 WaitForEncodedFrame(timestamp_ms);
5891 }
5892 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5893 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5894
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005895 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005896
5897 // Insert frame should trigger high BW and release quality limitation.
5898 timestamp_ms += kFrameIntervalMs;
5899 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5900 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005901 // The ramp-up code involves the adaptation queue, give it time to execute.
5902 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005903 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005904 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005905
5906 // Frame should not be adapted.
5907 timestamp_ms += kFrameIntervalMs;
5908 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5909 WaitForEncodedFrame(kWidth, kHeight);
5910 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5911
5912 video_stream_encoder_->Stop();
5913}
5914
mflodmancc3d4422017-08-03 08:27:51 -07005915TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005916 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005917 webrtc::test::ScopedFieldTrials field_trials(
5918 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005919 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005920 source.set_adaptation_enabled(true);
5921 video_stream_encoder_->SetSource(&source,
5922 DegradationPreference::MAINTAIN_FRAMERATE);
5923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5924 DataRate::BitsPerSec(kTargetBitrateBps),
5925 DataRate::BitsPerSec(kTargetBitrateBps),
5926 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5927 fake_encoder_.SetQp(kQpHigh + 1);
5928 const int kWidth = 1280;
5929 const int kHeight = 720;
5930 const int64_t kFrameIntervalMs = 100;
5931 int64_t timestamp_ms = kFrameIntervalMs;
5932 for (size_t i = 1; i <= 100; i++) {
5933 timestamp_ms += kFrameIntervalMs;
5934 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5935 WaitForEncodedFrame(timestamp_ms);
5936 }
5937 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5938 // for the first time.
5939 // TODO(eshr): We should avoid these waits by using threads with simulated
5940 // time.
5941 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5942 2000 * 2.5 * 2);
5943 timestamp_ms += kFrameIntervalMs;
5944 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5945 WaitForEncodedFrame(timestamp_ms);
5946 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5947 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5948 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5949
5950 // Disable Quality scaling by turning off scaler on the encoder and
5951 // reconfiguring.
5952 fake_encoder_.SetQualityScaling(false);
5953 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5954 kMaxPayloadLength);
5955 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005956 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005957 // Since we turned off the quality scaler, the adaptations made by it are
5958 // removed.
5959 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5960 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5961
5962 video_stream_encoder_->Stop();
5963}
5964
5965TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005966 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5967 const int kTooSmallWidth = 10;
5968 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005970 DataRate::BitsPerSec(kTargetBitrateBps),
5971 DataRate::BitsPerSec(kTargetBitrateBps),
5972 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005973
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005974 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005975 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005976 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005977 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005978 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005979 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5980
5981 // Trigger adapt down, too small frame, expect no change.
5982 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005983 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005984 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005985 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005986 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5987 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5988
mflodmancc3d4422017-08-03 08:27:51 -07005989 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005990}
5991
mflodmancc3d4422017-08-03 08:27:51 -07005992TEST_F(VideoStreamEncoderTest,
5993 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005994 const int kTooSmallWidth = 10;
5995 const int kTooSmallHeight = 10;
5996 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005997 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005998 DataRate::BitsPerSec(kTargetBitrateBps),
5999 DataRate::BitsPerSec(kTargetBitrateBps),
6000 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006001
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006002 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006003 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006004 video_stream_encoder_->SetSource(&source,
6005 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006006 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006007 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6008 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6009
6010 // Trigger adapt down, expect limited framerate.
6011 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006012 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006013 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006014 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006015 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6016 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6017 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6018
6019 // Trigger adapt down, too small frame, expect no change.
6020 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006021 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006022 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006023 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6025 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6026 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6027
mflodmancc3d4422017-08-03 08:27:51 -07006028 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006029}
6030
mflodmancc3d4422017-08-03 08:27:51 -07006031TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006032 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006034 DataRate::BitsPerSec(kTargetBitrateBps),
6035 DataRate::BitsPerSec(kTargetBitrateBps),
6036 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006037 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006038 const int kFrameWidth = 1280;
6039 const int kFrameHeight = 720;
6040 video_source_.IncomingCapturedFrame(
6041 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006042 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006043 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006044}
6045
sprangb1ca0732017-02-01 08:38:12 -08006046// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006047TEST_F(VideoStreamEncoderTest,
6048 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006050 DataRate::BitsPerSec(kTargetBitrateBps),
6051 DataRate::BitsPerSec(kTargetBitrateBps),
6052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006053
6054 const int kFrameWidth = 1280;
6055 const int kFrameHeight = 720;
6056 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006057 // requested by
6058 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006059 video_source_.set_adaptation_enabled(true);
6060
6061 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006062 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006063 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006064
6065 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006066 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006067 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006068 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006069 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006070
asaperssonfab67072017-04-04 05:51:49 -07006071 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006072 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006073 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006074 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006075 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006076
mflodmancc3d4422017-08-03 08:27:51 -07006077 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006078}
sprangfe627f32017-03-29 08:24:59 -07006079
mflodmancc3d4422017-08-03 08:27:51 -07006080TEST_F(VideoStreamEncoderTest,
6081 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006082 const int kFrameWidth = 1280;
6083 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006084
Henrik Boström381d1092020-05-12 18:49:07 +02006085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006086 DataRate::BitsPerSec(kTargetBitrateBps),
6087 DataRate::BitsPerSec(kTargetBitrateBps),
6088 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006089 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006090 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006091 video_source_.set_adaptation_enabled(true);
6092
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006093 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006094
6095 video_source_.IncomingCapturedFrame(
6096 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006097 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006098
6099 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006100 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006101
6102 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006103 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006104 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006105 video_source_.IncomingCapturedFrame(
6106 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006107 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006108 }
6109
6110 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006111 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006112 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006113 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006114 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006115 video_source_.IncomingCapturedFrame(
6116 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006117 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006118 ++num_frames_dropped;
6119 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006120 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006121 }
6122 }
6123
sprang4847ae62017-06-27 07:06:52 -07006124 // Add some slack to account for frames dropped by the frame dropper.
6125 const int kErrorMargin = 1;
6126 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006127 kErrorMargin);
6128
6129 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006130 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006131 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006132 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006133 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006134 video_source_.IncomingCapturedFrame(
6135 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006136 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006137 ++num_frames_dropped;
6138 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006139 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006140 }
6141 }
sprang4847ae62017-06-27 07:06:52 -07006142 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006143 kErrorMargin);
6144
6145 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006146 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006147 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006148 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006149 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006150 video_source_.IncomingCapturedFrame(
6151 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006152 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006153 ++num_frames_dropped;
6154 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006155 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006156 }
6157 }
sprang4847ae62017-06-27 07:06:52 -07006158 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006159 kErrorMargin);
6160
6161 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006162 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006163 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006164 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006165 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006166 video_source_.IncomingCapturedFrame(
6167 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006168 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006169 ++num_frames_dropped;
6170 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006171 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006172 }
6173 }
6174 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6175
mflodmancc3d4422017-08-03 08:27:51 -07006176 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006177}
6178
mflodmancc3d4422017-08-03 08:27:51 -07006179TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006180 const int kFramerateFps = 5;
6181 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006182 const int kFrameWidth = 1280;
6183 const int kFrameHeight = 720;
6184
sprang4847ae62017-06-27 07:06:52 -07006185 // Reconfigure encoder with two temporal layers and screensharing, which will
6186 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006187 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006188
Henrik Boström381d1092020-05-12 18:49:07 +02006189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006190 DataRate::BitsPerSec(kTargetBitrateBps),
6191 DataRate::BitsPerSec(kTargetBitrateBps),
6192 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006193 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006194 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006195 video_source_.set_adaptation_enabled(true);
6196
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006197 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006198
6199 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006200 rtc::VideoSinkWants last_wants;
6201 do {
6202 last_wants = video_source_.sink_wants();
6203
sprangc5d62e22017-04-02 23:53:04 -07006204 // Insert frames to get a new fps estimate...
6205 for (int j = 0; j < kFramerateFps; ++j) {
6206 video_source_.IncomingCapturedFrame(
6207 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006208 if (video_source_.last_sent_width()) {
6209 sink_.WaitForEncodedFrame(timestamp_ms);
6210 }
sprangc5d62e22017-04-02 23:53:04 -07006211 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006212 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006213 }
6214 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006215 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006216 } while (video_source_.sink_wants().max_framerate_fps <
6217 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006218
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006219 EXPECT_THAT(video_source_.sink_wants(),
6220 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006221
mflodmancc3d4422017-08-03 08:27:51 -07006222 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006223}
asaperssonf7e294d2017-06-13 23:25:22 -07006224
mflodmancc3d4422017-08-03 08:27:51 -07006225TEST_F(VideoStreamEncoderTest,
6226 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006227 const int kWidth = 1280;
6228 const int kHeight = 720;
6229 const int64_t kFrameIntervalMs = 150;
6230 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006232 DataRate::BitsPerSec(kTargetBitrateBps),
6233 DataRate::BitsPerSec(kTargetBitrateBps),
6234 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006235
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006236 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006237 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006238 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006239 video_stream_encoder_->SetSource(&source,
6240 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006241 timestamp_ms += kFrameIntervalMs;
6242 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006243 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006244 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006245 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6247 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6248
6249 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006250 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006251 timestamp_ms += kFrameIntervalMs;
6252 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006253 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006254 EXPECT_THAT(source.sink_wants(),
6255 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006256 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6258 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6259
6260 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006261 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006262 timestamp_ms += kFrameIntervalMs;
6263 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006264 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006265 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006266 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6267 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6268 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6269
6270 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006271 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006272 timestamp_ms += kFrameIntervalMs;
6273 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006274 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006275 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6277 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6278 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6279
6280 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006281 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006282 timestamp_ms += kFrameIntervalMs;
6283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006284 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006285 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6287 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6288 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6289
6290 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006292 timestamp_ms += kFrameIntervalMs;
6293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006294 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006295 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006296 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6297 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6298 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6299
6300 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006301 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006302 timestamp_ms += kFrameIntervalMs;
6303 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006304 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006305 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006306 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6308 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6309
6310 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006311 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006312 timestamp_ms += kFrameIntervalMs;
6313 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006314 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006315 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006316 rtc::VideoSinkWants last_wants = source.sink_wants();
6317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6318 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6319 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6320
6321 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006322 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006323 timestamp_ms += kFrameIntervalMs;
6324 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006325 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006326 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6329 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6330
Evan Shrubsole64469032020-06-11 10:45:29 +02006331 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006332 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006333 timestamp_ms += kFrameIntervalMs;
6334 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006335 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006336 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6339 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6340
6341 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006342 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006343 timestamp_ms += kFrameIntervalMs;
6344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006345 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006346 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6349 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6350
6351 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006352 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006353 timestamp_ms += kFrameIntervalMs;
6354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006355 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006356 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6359 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6360
6361 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006363 timestamp_ms += kFrameIntervalMs;
6364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006365 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006366 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006367 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6368 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6369 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6370
6371 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006372 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006373 timestamp_ms += kFrameIntervalMs;
6374 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006375 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006376 EXPECT_THAT(source.sink_wants(), FpsMax());
6377 EXPECT_EQ(source.sink_wants().max_pixel_count,
6378 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6380 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6381 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6382
6383 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006385 timestamp_ms += kFrameIntervalMs;
6386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006387 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006388 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6391 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6392
Åsa Persson30ab0152019-08-27 12:22:33 +02006393 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006394 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006395 timestamp_ms += kFrameIntervalMs;
6396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006397 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006398 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006399 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6402 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6403
6404 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006405 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006406 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006407 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6408
mflodmancc3d4422017-08-03 08:27:51 -07006409 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006410}
6411
mflodmancc3d4422017-08-03 08:27:51 -07006412TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006413 const int kWidth = 1280;
6414 const int kHeight = 720;
6415 const int64_t kFrameIntervalMs = 150;
6416 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006418 DataRate::BitsPerSec(kTargetBitrateBps),
6419 DataRate::BitsPerSec(kTargetBitrateBps),
6420 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006421
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006422 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006423 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006424 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006425 video_stream_encoder_->SetSource(&source,
6426 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006427 timestamp_ms += kFrameIntervalMs;
6428 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006429 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006430 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006431 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6432 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6433 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6434 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6435 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6436 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6437
6438 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006439 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006440 timestamp_ms += kFrameIntervalMs;
6441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006442 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006443 EXPECT_THAT(source.sink_wants(),
6444 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006445 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6447 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6448 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6449 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6450 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6451
6452 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006453 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006454 timestamp_ms += kFrameIntervalMs;
6455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006456 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006457 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6459 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6460 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6461 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6462 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6463 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6464
6465 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006466 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006467 timestamp_ms += kFrameIntervalMs;
6468 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006469 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006470 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006471 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6473 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6474 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6475 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6476 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6477
Evan Shrubsole64469032020-06-11 10:45:29 +02006478 // Trigger cpu adapt up, expect no change since QP is most limited.
6479 {
6480 // Store current sink wants since we expect no change and if there is no
6481 // change then last_wants() is not updated.
6482 auto previous_sink_wants = source.sink_wants();
6483 video_stream_encoder_->TriggerCpuUnderuse();
6484 timestamp_ms += kFrameIntervalMs;
6485 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6486 WaitForEncodedFrame(timestamp_ms);
6487 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6488 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6489 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6490 }
6491
6492 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6493 video_stream_encoder_->TriggerQualityHigh();
6494 timestamp_ms += kFrameIntervalMs;
6495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6496 WaitForEncodedFrame(timestamp_ms);
6497 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6500 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6501 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6503 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6504
6505 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6506 // expect increased resolution (960x540@30fps).
6507 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006508 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006509 timestamp_ms += kFrameIntervalMs;
6510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006511 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006512 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6515 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6516 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6517 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006518 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006519
Evan Shrubsole64469032020-06-11 10:45:29 +02006520 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6521 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006522 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006523 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006524 timestamp_ms += kFrameIntervalMs;
6525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006526 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006527 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006528 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6533 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006534 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006535
6536 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006537 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006538 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006539 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006540 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006541
mflodmancc3d4422017-08-03 08:27:51 -07006542 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006543}
6544
mflodmancc3d4422017-08-03 08:27:51 -07006545TEST_F(VideoStreamEncoderTest,
6546 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006547 const int kWidth = 640;
6548 const int kHeight = 360;
6549 const int kFpsLimit = 15;
6550 const int64_t kFrameIntervalMs = 150;
6551 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006553 DataRate::BitsPerSec(kTargetBitrateBps),
6554 DataRate::BitsPerSec(kTargetBitrateBps),
6555 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006556
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006557 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006558 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006559 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006560 video_stream_encoder_->SetSource(&source,
6561 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006565 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6569 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6570 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6571 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6572
6573 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006574 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006575 timestamp_ms += kFrameIntervalMs;
6576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006577 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006578 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006579 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6580 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6581 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6583 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6584 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6585
6586 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006587 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006588 timestamp_ms += kFrameIntervalMs;
6589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006590 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006591 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006594 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6595 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6596 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6597 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598
Evan Shrubsole64469032020-06-11 10:45:29 +02006599 // Trigger cpu adapt up, expect no change because quality is most limited.
6600 {
6601 auto previous_sink_wants = source.sink_wants();
6602 // Store current sink wants since we expect no change ind if there is no
6603 // change then last__wants() is not updated.
6604 video_stream_encoder_->TriggerCpuUnderuse();
6605 timestamp_ms += kFrameIntervalMs;
6606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6607 WaitForEncodedFrame(timestamp_ms);
6608 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6609 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6610 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611 }
6612
6613 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6614 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006615 timestamp_ms += kFrameIntervalMs;
6616 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006617 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006618 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6621 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006622 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6623 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6624 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006625
Evan Shrubsole64469032020-06-11 10:45:29 +02006626 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006627 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006628 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006631 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006632 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6634 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6635 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6636 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6637 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006638 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006639
6640 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006641 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006642 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006643 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006644 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006645
mflodmancc3d4422017-08-03 08:27:51 -07006646 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006647}
6648
mflodmancc3d4422017-08-03 08:27:51 -07006649TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006650 const int kFrameWidth = 1920;
6651 const int kFrameHeight = 1080;
6652 // 3/4 of 1920.
6653 const int kAdaptedFrameWidth = 1440;
6654 // 3/4 of 1080 rounded down to multiple of 4.
6655 const int kAdaptedFrameHeight = 808;
6656 const int kFramerate = 24;
6657
Henrik Boström381d1092020-05-12 18:49:07 +02006658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006659 DataRate::BitsPerSec(kTargetBitrateBps),
6660 DataRate::BitsPerSec(kTargetBitrateBps),
6661 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006662 // Trigger reconfigure encoder (without resetting the entire instance).
6663 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006664 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6665 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006666 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006667 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006668 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006669 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006670 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006671 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006672
6673 video_source_.set_adaptation_enabled(true);
6674
6675 video_source_.IncomingCapturedFrame(
6676 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006677 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006678
6679 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006680 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006681 video_source_.IncomingCapturedFrame(
6682 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006683 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006684
mflodmancc3d4422017-08-03 08:27:51 -07006685 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006686}
6687
mflodmancc3d4422017-08-03 08:27:51 -07006688TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006689 const int kFrameWidth = 1280;
6690 const int kFrameHeight = 720;
6691 const int kLowFps = 2;
6692 const int kHighFps = 30;
6693
Henrik Boström381d1092020-05-12 18:49:07 +02006694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006695 DataRate::BitsPerSec(kTargetBitrateBps),
6696 DataRate::BitsPerSec(kTargetBitrateBps),
6697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006698
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006699 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006700 max_framerate_ = kLowFps;
6701
6702 // Insert 2 seconds of 2fps video.
6703 for (int i = 0; i < kLowFps * 2; ++i) {
6704 video_source_.IncomingCapturedFrame(
6705 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6706 WaitForEncodedFrame(timestamp_ms);
6707 timestamp_ms += 1000 / kLowFps;
6708 }
6709
6710 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006712 DataRate::BitsPerSec(kTargetBitrateBps),
6713 DataRate::BitsPerSec(kTargetBitrateBps),
6714 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006715 video_source_.IncomingCapturedFrame(
6716 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6717 WaitForEncodedFrame(timestamp_ms);
6718 timestamp_ms += 1000 / kLowFps;
6719
6720 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6721
6722 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006723 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006724 const int kFrameIntervalMs = 1000 / kHighFps;
6725 max_framerate_ = kHighFps;
6726 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6727 video_source_.IncomingCapturedFrame(
6728 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6729 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6730 // be dropped if the encoder hans't been updated with the new higher target
6731 // framerate yet, causing it to overshoot the target bitrate and then
6732 // suffering the wrath of the media optimizer.
6733 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6734 timestamp_ms += kFrameIntervalMs;
6735 }
6736
6737 // Don expect correct measurement just yet, but it should be higher than
6738 // before.
6739 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6740
mflodmancc3d4422017-08-03 08:27:51 -07006741 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006742}
6743
mflodmancc3d4422017-08-03 08:27:51 -07006744TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006745 const int kFrameWidth = 1280;
6746 const int kFrameHeight = 720;
6747 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006748 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006749 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006750 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006751
Henrik Boström381d1092020-05-12 18:49:07 +02006752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006753 DataRate::BitsPerSec(kTargetBitrateBps),
6754 DataRate::BitsPerSec(kTargetBitrateBps),
6755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006756 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006757
6758 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006759 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006760 video_source_.IncomingCapturedFrame(
6761 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6762 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006763 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006764
6765 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6767 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6768 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006769
6770 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006771 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006772 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006773
Per Kjellanderdcef6412020-10-07 15:09:05 +02006774 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006775 video_source_.IncomingCapturedFrame(
6776 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6777 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006778 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006779
mflodmancc3d4422017-08-03 08:27:51 -07006780 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006781}
ilnik6b826ef2017-06-16 06:53:48 -07006782
Niels Möller4db138e2018-04-19 09:04:13 +02006783TEST_F(VideoStreamEncoderTest,
6784 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6785 const int kFrameWidth = 1280;
6786 const int kFrameHeight = 720;
6787 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006789 DataRate::BitsPerSec(kTargetBitrateBps),
6790 DataRate::BitsPerSec(kTargetBitrateBps),
6791 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006792 video_source_.IncomingCapturedFrame(
6793 CreateFrame(1, kFrameWidth, kFrameHeight));
6794 WaitForEncodedFrame(1);
6795 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6796 .low_encode_usage_threshold_percent,
6797 default_options.low_encode_usage_threshold_percent);
6798 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6799 .high_encode_usage_threshold_percent,
6800 default_options.high_encode_usage_threshold_percent);
6801 video_stream_encoder_->Stop();
6802}
6803
6804TEST_F(VideoStreamEncoderTest,
6805 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6806 const int kFrameWidth = 1280;
6807 const int kFrameHeight = 720;
6808 CpuOveruseOptions hardware_options;
6809 hardware_options.low_encode_usage_threshold_percent = 150;
6810 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006811 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006812
Henrik Boström381d1092020-05-12 18:49:07 +02006813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006814 DataRate::BitsPerSec(kTargetBitrateBps),
6815 DataRate::BitsPerSec(kTargetBitrateBps),
6816 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006817 video_source_.IncomingCapturedFrame(
6818 CreateFrame(1, kFrameWidth, kFrameHeight));
6819 WaitForEncodedFrame(1);
6820 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6821 .low_encode_usage_threshold_percent,
6822 hardware_options.low_encode_usage_threshold_percent);
6823 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6824 .high_encode_usage_threshold_percent,
6825 hardware_options.high_encode_usage_threshold_percent);
6826 video_stream_encoder_->Stop();
6827}
6828
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006829TEST_F(VideoStreamEncoderTest,
6830 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6831 const int kFrameWidth = 1280;
6832 const int kFrameHeight = 720;
6833
6834 const CpuOveruseOptions default_options;
6835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6836 DataRate::BitsPerSec(kTargetBitrateBps),
6837 DataRate::BitsPerSec(kTargetBitrateBps),
6838 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6839 video_source_.IncomingCapturedFrame(
6840 CreateFrame(1, kFrameWidth, kFrameHeight));
6841 WaitForEncodedFrame(1);
6842 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6843 .low_encode_usage_threshold_percent,
6844 default_options.low_encode_usage_threshold_percent);
6845 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6846 .high_encode_usage_threshold_percent,
6847 default_options.high_encode_usage_threshold_percent);
6848
6849 CpuOveruseOptions hardware_options;
6850 hardware_options.low_encode_usage_threshold_percent = 150;
6851 hardware_options.high_encode_usage_threshold_percent = 200;
6852 fake_encoder_.SetIsHardwareAccelerated(true);
6853
6854 video_source_.IncomingCapturedFrame(
6855 CreateFrame(2, kFrameWidth, kFrameHeight));
6856 WaitForEncodedFrame(2);
6857
6858 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6859 .low_encode_usage_threshold_percent,
6860 hardware_options.low_encode_usage_threshold_percent);
6861 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6862 .high_encode_usage_threshold_percent,
6863 hardware_options.high_encode_usage_threshold_percent);
6864
6865 video_stream_encoder_->Stop();
6866}
6867
Niels Möller6bb5ab92019-01-11 11:11:10 +01006868TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6869 const int kFrameWidth = 320;
6870 const int kFrameHeight = 240;
6871 const int kFps = 30;
6872 const int kTargetBitrateBps = 120000;
6873 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6874
Henrik Boström381d1092020-05-12 18:49:07 +02006875 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006876 DataRate::BitsPerSec(kTargetBitrateBps),
6877 DataRate::BitsPerSec(kTargetBitrateBps),
6878 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006879
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006880 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006881 max_framerate_ = kFps;
6882
6883 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6884 fake_encoder_.SimulateOvershoot(1.0);
6885 int num_dropped = 0;
6886 for (int i = 0; i < kNumFramesInRun; ++i) {
6887 video_source_.IncomingCapturedFrame(
6888 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6889 // Wait up to two frame durations for a frame to arrive.
6890 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6891 ++num_dropped;
6892 }
6893 timestamp_ms += 1000 / kFps;
6894 }
6895
Erik Språnga8d48ab2019-02-08 14:17:40 +01006896 // Framerate should be measured to be near the expected target rate.
6897 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6898
6899 // Frame drops should be within 5% of expected 0%.
6900 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006901
6902 // Make encoder produce frames at double the expected bitrate during 3 seconds
6903 // of video, verify number of drops. Rate needs to be slightly changed in
6904 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006905 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006906 const RateControlSettings trials =
6907 RateControlSettings::ParseFromFieldTrials();
6908 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006909 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006910 // frame dropping since the adjuter will try to just lower the target
6911 // bitrate rather than drop frames. If network headroom can be used, it
6912 // doesn't push back as hard so we don't need quite as much overshoot.
6913 // These numbers are unfortunately a bit magical but there's not trivial
6914 // way to algebraically infer them.
6915 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6916 overshoot_factor = 2.4;
6917 } else {
6918 overshoot_factor = 4.0;
6919 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006920 }
6921 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006923 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6924 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6925 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006926 num_dropped = 0;
6927 for (int i = 0; i < kNumFramesInRun; ++i) {
6928 video_source_.IncomingCapturedFrame(
6929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6930 // Wait up to two frame durations for a frame to arrive.
6931 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6932 ++num_dropped;
6933 }
6934 timestamp_ms += 1000 / kFps;
6935 }
6936
Henrik Boström381d1092020-05-12 18:49:07 +02006937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006938 DataRate::BitsPerSec(kTargetBitrateBps),
6939 DataRate::BitsPerSec(kTargetBitrateBps),
6940 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006941
6942 // Target framerate should be still be near the expected target, despite
6943 // the frame drops.
6944 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6945
6946 // Frame drops should be within 5% of expected 50%.
6947 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006948
6949 video_stream_encoder_->Stop();
6950}
6951
6952TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6953 const int kFrameWidth = 320;
6954 const int kFrameHeight = 240;
6955 const int kActualInputFps = 24;
6956 const int kTargetBitrateBps = 120000;
6957
6958 ASSERT_GT(max_framerate_, kActualInputFps);
6959
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006960 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006961 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006962 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006963 DataRate::BitsPerSec(kTargetBitrateBps),
6964 DataRate::BitsPerSec(kTargetBitrateBps),
6965 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006966
6967 // Insert 3 seconds of video, with an input fps lower than configured max.
6968 for (int i = 0; i < kActualInputFps * 3; ++i) {
6969 video_source_.IncomingCapturedFrame(
6970 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6971 // Wait up to two frame durations for a frame to arrive.
6972 WaitForEncodedFrame(timestamp_ms);
6973 timestamp_ms += 1000 / kActualInputFps;
6974 }
6975
6976 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6977
6978 video_stream_encoder_->Stop();
6979}
6980
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006981TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006982 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006984 DataRate::BitsPerSec(kTargetBitrateBps),
6985 DataRate::BitsPerSec(kTargetBitrateBps),
6986 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006987
6988 fake_encoder_.BlockNextEncode();
6989 video_source_.IncomingCapturedFrame(
6990 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6991 WaitForEncodedFrame(1);
6992 // On the very first frame full update should be forced.
6993 rect = fake_encoder_.GetLastUpdateRect();
6994 EXPECT_EQ(rect.offset_x, 0);
6995 EXPECT_EQ(rect.offset_y, 0);
6996 EXPECT_EQ(rect.height, codec_height_);
6997 EXPECT_EQ(rect.width, codec_width_);
6998 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6999 // call to ContinueEncode.
7000 video_source_.IncomingCapturedFrame(
7001 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7002 ExpectDroppedFrame();
7003 video_source_.IncomingCapturedFrame(
7004 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7005 ExpectDroppedFrame();
7006 fake_encoder_.ContinueEncode();
7007 WaitForEncodedFrame(3);
7008 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7009 rect = fake_encoder_.GetLastUpdateRect();
7010 EXPECT_EQ(rect.offset_x, 1);
7011 EXPECT_EQ(rect.offset_y, 0);
7012 EXPECT_EQ(rect.width, 10);
7013 EXPECT_EQ(rect.height, 1);
7014
7015 video_source_.IncomingCapturedFrame(
7016 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7017 WaitForEncodedFrame(4);
7018 // Previous frame was encoded, so no accumulation should happen.
7019 rect = fake_encoder_.GetLastUpdateRect();
7020 EXPECT_EQ(rect.offset_x, 0);
7021 EXPECT_EQ(rect.offset_y, 0);
7022 EXPECT_EQ(rect.width, 1);
7023 EXPECT_EQ(rect.height, 1);
7024
7025 video_stream_encoder_->Stop();
7026}
7027
Erik Språngd7329ca2019-02-21 21:19:53 +01007028TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007030 DataRate::BitsPerSec(kTargetBitrateBps),
7031 DataRate::BitsPerSec(kTargetBitrateBps),
7032 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007033
7034 // First frame is always keyframe.
7035 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7036 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007037 EXPECT_THAT(
7038 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007039 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007040
7041 // Insert delta frame.
7042 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7043 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007044 EXPECT_THAT(
7045 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007046 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007047
7048 // Request next frame be a key-frame.
7049 video_stream_encoder_->SendKeyFrame();
7050 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7051 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007052 EXPECT_THAT(
7053 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007054 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007055
7056 video_stream_encoder_->Stop();
7057}
7058
7059TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7060 // Setup simulcast with three streams.
7061 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007063 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7064 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7065 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007066 // Wait for all three layers before triggering event.
7067 sink_.SetNumExpectedLayers(3);
7068
7069 // First frame is always keyframe.
7070 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7071 WaitForEncodedFrame(1);
7072 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007073 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7074 VideoFrameType::kVideoFrameKey,
7075 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007076
7077 // Insert delta frame.
7078 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7079 WaitForEncodedFrame(2);
7080 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007081 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7082 VideoFrameType::kVideoFrameDelta,
7083 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007084
7085 // Request next frame be a key-frame.
7086 // Only first stream is configured to produce key-frame.
7087 video_stream_encoder_->SendKeyFrame();
7088 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7089 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007090
7091 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7092 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007093 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007094 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007095 VideoFrameType::kVideoFrameKey,
7096 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007097
7098 video_stream_encoder_->Stop();
7099}
7100
7101TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7102 // Configure internal source factory and setup test again.
7103 encoder_factory_.SetHasInternalSource(true);
7104 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007106 DataRate::BitsPerSec(kTargetBitrateBps),
7107 DataRate::BitsPerSec(kTargetBitrateBps),
7108 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007109
7110 // Call encoder directly, simulating internal source where encoded frame
7111 // callback in VideoStreamEncoder is called despite no OnFrame().
7112 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7113 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007114 EXPECT_THAT(
7115 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007116 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007117
Niels Möller8f7ce222019-03-21 15:43:58 +01007118 const std::vector<VideoFrameType> kDeltaFrame = {
7119 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007120 // Need to set timestamp manually since manually for injected frame.
7121 VideoFrame frame = CreateFrame(101, nullptr);
7122 frame.set_timestamp(101);
7123 fake_encoder_.InjectFrame(frame, false);
7124 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007125 EXPECT_THAT(
7126 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007127 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007128
7129 // Request key-frame. The forces a dummy frame down into the encoder.
7130 fake_encoder_.ExpectNullFrame();
7131 video_stream_encoder_->SendKeyFrame();
7132 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007133 EXPECT_THAT(
7134 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007135 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007136
7137 video_stream_encoder_->Stop();
7138}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007139
7140TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7141 // Configure internal source factory and setup test again.
7142 encoder_factory_.SetHasInternalSource(true);
7143 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007144 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007145 DataRate::BitsPerSec(kTargetBitrateBps),
7146 DataRate::BitsPerSec(kTargetBitrateBps),
7147 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007148
7149 int64_t timestamp = 1;
7150 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007151 image.capture_time_ms_ = ++timestamp;
7152 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7153 const int64_t kEncodeFinishDelayMs = 10;
7154 image.timing_.encode_start_ms = timestamp;
7155 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007156 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007157 // Wait for frame without incrementing clock.
7158 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7159 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7160 // capture timestamp should be kEncodeFinishDelayMs in the past.
7161 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007162 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007163
7164 video_stream_encoder_->Stop();
7165}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007166
7167TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007168 // SPS contains VUI with restrictions on the maximum number of reordered
7169 // pictures, there is no need to rewrite the bitstream to enable faster
7170 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007171 ResetEncoder("H264", 1, 1, 1, false);
7172
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7174 DataRate::BitsPerSec(kTargetBitrateBps),
7175 DataRate::BitsPerSec(kTargetBitrateBps),
7176 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7177 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007178
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007179 fake_encoder_.SetEncodedImageData(
7180 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007181
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007182 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7183 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007184
7185 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7186 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007187
7188 video_stream_encoder_->Stop();
7189}
7190
7191TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007192 // SPS does not contain VUI, the bitstream is will be rewritten with added
7193 // VUI with restrictions on the maximum number of reordered pictures to
7194 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007195 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7196 0x00, 0x00, 0x03, 0x03, 0xF4,
7197 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007198 ResetEncoder("H264", 1, 1, 1, false);
7199
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007200 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7201 DataRate::BitsPerSec(kTargetBitrateBps),
7202 DataRate::BitsPerSec(kTargetBitrateBps),
7203 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7204 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007205
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007206 fake_encoder_.SetEncodedImageData(
7207 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007208
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007209 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7210 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007211
7212 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7213 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007214
7215 video_stream_encoder_->Stop();
7216}
7217
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007218TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7219 const int kFrameWidth = 1280;
7220 const int kFrameHeight = 720;
7221 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7222
Henrik Boström381d1092020-05-12 18:49:07 +02007223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007224 DataRate::BitsPerSec(kTargetBitrateBps),
7225 DataRate::BitsPerSec(kTargetBitrateBps),
7226 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007227 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7228
7229 // Insert a first video frame. It should be dropped because of downscale in
7230 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007231 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007232 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7233 frame.set_rotation(kVideoRotation_270);
7234 video_source_.IncomingCapturedFrame(frame);
7235
7236 ExpectDroppedFrame();
7237
7238 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007239 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007240 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7241 frame.set_rotation(kVideoRotation_90);
7242 video_source_.IncomingCapturedFrame(frame);
7243
7244 WaitForEncodedFrame(timestamp_ms);
7245 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7246
7247 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007248 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007249 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7250 frame.set_rotation(kVideoRotation_180);
7251 video_source_.IncomingCapturedFrame(frame);
7252
7253 WaitForEncodedFrame(timestamp_ms);
7254 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7255
7256 video_stream_encoder_->Stop();
7257}
7258
Erik Språng5056af02019-09-02 15:53:11 +02007259TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7260 const int kFrameWidth = 320;
7261 const int kFrameHeight = 180;
7262
7263 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007265 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7266 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7267 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007268 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007269 /*rtt_ms=*/0,
7270 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007271
7272 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007273 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007274 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7275 frame.set_rotation(kVideoRotation_270);
7276 video_source_.IncomingCapturedFrame(frame);
7277 WaitForEncodedFrame(timestamp_ms);
7278
7279 // Set a target rate below the minimum allowed by the codec settings.
7280 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007281 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7282 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007284 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007285 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007286 /*link_allocation=*/target_rate,
7287 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007288 /*rtt_ms=*/0,
7289 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7291
7292 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7293 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7294 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007295 DataRate allocation_sum =
7296 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007297 EXPECT_EQ(min_rate, allocation_sum);
7298 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7299
7300 video_stream_encoder_->Stop();
7301}
7302
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007303TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007305 DataRate::BitsPerSec(kTargetBitrateBps),
7306 DataRate::BitsPerSec(kTargetBitrateBps),
7307 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007308 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007309 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007310 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7311 WaitForEncodedFrame(1);
7312
7313 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7314 ASSERT_TRUE(prev_rate_settings.has_value());
7315 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7316 kDefaultFramerate);
7317
7318 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7319 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7320 timestamp_ms += 1000 / kDefaultFramerate;
7321 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7322 WaitForEncodedFrame(timestamp_ms);
7323 }
7324 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7325 kDefaultFramerate);
7326 // Capture larger frame to trigger a reconfigure.
7327 codec_height_ *= 2;
7328 codec_width_ *= 2;
7329 timestamp_ms += 1000 / kDefaultFramerate;
7330 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7331 WaitForEncodedFrame(timestamp_ms);
7332
7333 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7334 auto current_rate_settings =
7335 fake_encoder_.GetAndResetLastRateControlSettings();
7336 // Ensure we have actually reconfigured twice
7337 // The rate settings should have been set again even though
7338 // they haven't changed.
7339 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007340 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007341
7342 video_stream_encoder_->Stop();
7343}
7344
philipeld9cc8c02019-09-16 14:53:40 +02007345struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007346 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7347 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7348 MOCK_METHOD(void,
7349 RequestEncoderSwitch,
7350 (const webrtc::SdpVideoFormat& format),
7351 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007352};
7353
philipel9b058032020-02-10 11:30:00 +01007354TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7355 constexpr int kDontCare = 100;
7356 StrictMock<MockEncoderSelector> encoder_selector;
7357 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7358 &fake_encoder_, &encoder_selector);
7359 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7360
7361 // Reset encoder for new configuration to take effect.
7362 ConfigureEncoder(video_encoder_config_.Copy());
7363
7364 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7365
7366 video_source_.IncomingCapturedFrame(
7367 CreateFrame(kDontCare, kDontCare, kDontCare));
7368 video_stream_encoder_->Stop();
7369
7370 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7371 // to it's factory, so in order for the encoder instance in the
7372 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7373 // reset the |video_stream_encoder_| here.
7374 video_stream_encoder_.reset();
7375}
7376
7377TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7378 constexpr int kDontCare = 100;
7379
7380 NiceMock<MockEncoderSelector> encoder_selector;
7381 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7382 video_send_config_.encoder_settings.encoder_switch_request_callback =
7383 &switch_callback;
7384 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7385 &fake_encoder_, &encoder_selector);
7386 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7387
7388 // Reset encoder for new configuration to take effect.
7389 ConfigureEncoder(video_encoder_config_.Copy());
7390
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007391 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007392 .WillByDefault(Return(SdpVideoFormat("AV1")));
7393 EXPECT_CALL(switch_callback,
7394 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7395 Field(&SdpVideoFormat::name, "AV1"))));
7396
Henrik Boström381d1092020-05-12 18:49:07 +02007397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007398 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7399 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7400 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007401 /*fraction_lost=*/0,
7402 /*rtt_ms=*/0,
7403 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007404 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007405
7406 video_stream_encoder_->Stop();
7407}
7408
7409TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7410 constexpr int kSufficientBitrateToNotDrop = 1000;
7411 constexpr int kDontCare = 100;
7412
7413 NiceMock<MockVideoEncoder> video_encoder;
7414 NiceMock<MockEncoderSelector> encoder_selector;
7415 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7416 video_send_config_.encoder_settings.encoder_switch_request_callback =
7417 &switch_callback;
7418 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7419 &video_encoder, &encoder_selector);
7420 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7421
7422 // Reset encoder for new configuration to take effect.
7423 ConfigureEncoder(video_encoder_config_.Copy());
7424
7425 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7426 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7427 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007428 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007429 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7430 /*stable_target_bitrate=*/
7431 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7432 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007433 /*fraction_lost=*/0,
7434 /*rtt_ms=*/0,
7435 /*cwnd_reduce_ratio=*/0);
7436
7437 ON_CALL(video_encoder, Encode(_, _))
7438 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7439 ON_CALL(encoder_selector, OnEncoderBroken())
7440 .WillByDefault(Return(SdpVideoFormat("AV2")));
7441
7442 rtc::Event encode_attempted;
7443 EXPECT_CALL(switch_callback,
7444 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7445 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7446 EXPECT_EQ(format.name, "AV2");
7447 encode_attempted.Set();
7448 });
7449
7450 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7451 encode_attempted.Wait(3000);
7452
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007453 AdvanceTime(TimeDelta::Millis(0));
7454
philipel9b058032020-02-10 11:30:00 +01007455 video_stream_encoder_->Stop();
7456
7457 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7458 // to it's factory, so in order for the encoder instance in the
7459 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7460 // reset the |video_stream_encoder_| here.
7461 video_stream_encoder_.reset();
7462}
7463
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007464TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007465 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007466 const int kFrameWidth = 320;
7467 const int kFrameHeight = 180;
7468
7469 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007470 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007472 /*target_bitrate=*/rate,
7473 /*stable_target_bitrate=*/rate,
7474 /*link_allocation=*/rate,
7475 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007476 /*rtt_ms=*/0,
7477 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007478
7479 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007480 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007481 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7482 frame.set_rotation(kVideoRotation_270);
7483 video_source_.IncomingCapturedFrame(frame);
7484 WaitForEncodedFrame(timestamp_ms);
7485 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7486
7487 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007488 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007490 /*target_bitrate=*/new_stable_rate,
7491 /*stable_target_bitrate=*/new_stable_rate,
7492 /*link_allocation=*/rate,
7493 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007494 /*rtt_ms=*/0,
7495 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007496 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7497 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7498 video_stream_encoder_->Stop();
7499}
7500
7501TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007502 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007503 const int kFrameWidth = 320;
7504 const int kFrameHeight = 180;
7505
7506 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007507 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007509 /*target_bitrate=*/rate,
7510 /*stable_target_bitrate=*/rate,
7511 /*link_allocation=*/rate,
7512 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007513 /*rtt_ms=*/0,
7514 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007515
7516 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007517 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007518 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7519 frame.set_rotation(kVideoRotation_270);
7520 video_source_.IncomingCapturedFrame(frame);
7521 WaitForEncodedFrame(timestamp_ms);
7522 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7523
7524 // Set a higher target rate without changing the link_allocation. Should not
7525 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007526 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007527 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007528 /*target_bitrate=*/rate,
7529 /*stable_target_bitrate=*/new_stable_rate,
7530 /*link_allocation=*/rate,
7531 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007532 /*rtt_ms=*/0,
7533 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007534 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7535 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7536 video_stream_encoder_->Stop();
7537}
7538
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007539TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7540 test::ScopedFieldTrials field_trials(
7541 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7542 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7543 const int kFramerateFps = 30;
7544 const int kWidth = 1920;
7545 const int kHeight = 1080;
7546 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7547 // Works on screenshare mode.
7548 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7549 // We rely on the automatic resolution adaptation, but we handle framerate
7550 // adaptation manually by mocking the stats proxy.
7551 video_source_.set_adaptation_enabled(true);
7552
7553 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007555 DataRate::BitsPerSec(kTargetBitrateBps),
7556 DataRate::BitsPerSec(kTargetBitrateBps),
7557 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007558 video_stream_encoder_->SetSource(&video_source_,
7559 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007560 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007561
7562 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7563 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7564
7565 // Pass enough frames with the full update to trigger animation detection.
7566 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007567 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007568 frame.set_ntp_time_ms(timestamp_ms);
7569 frame.set_timestamp_us(timestamp_ms * 1000);
7570 video_source_.IncomingCapturedFrame(frame);
7571 WaitForEncodedFrame(timestamp_ms);
7572 }
7573
7574 // Resolution should be limited.
7575 rtc::VideoSinkWants expected;
7576 expected.max_framerate_fps = kFramerateFps;
7577 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007578 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007579
7580 // Pass one frame with no known update.
7581 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007582 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007583 frame.set_ntp_time_ms(timestamp_ms);
7584 frame.set_timestamp_us(timestamp_ms * 1000);
7585 frame.clear_update_rect();
7586
7587 video_source_.IncomingCapturedFrame(frame);
7588 WaitForEncodedFrame(timestamp_ms);
7589
7590 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007591 EXPECT_THAT(video_source_.sink_wants(),
7592 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007593
7594 video_stream_encoder_->Stop();
7595}
7596
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007597TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7598 const int kWidth = 720; // 540p adapted down.
7599 const int kHeight = 405;
7600 const int kNumFrames = 3;
7601 // Works on screenshare mode.
7602 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7603 /*num_spatial_layers=*/2, /*screenshare=*/true);
7604
7605 video_source_.set_adaptation_enabled(true);
7606
7607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7608 DataRate::BitsPerSec(kTargetBitrateBps),
7609 DataRate::BitsPerSec(kTargetBitrateBps),
7610 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7611
7612 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7613
7614 // Pass enough frames with the full update to trigger animation detection.
7615 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007616 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007617 frame.set_ntp_time_ms(timestamp_ms);
7618 frame.set_timestamp_us(timestamp_ms * 1000);
7619 video_source_.IncomingCapturedFrame(frame);
7620 WaitForEncodedFrame(timestamp_ms);
7621 }
7622
7623 video_stream_encoder_->Stop();
7624}
7625
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007626TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7627 const float downscale_factors[] = {4.0, 2.0, 1.0};
7628 const int number_layers =
7629 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7630 VideoEncoderConfig config;
7631 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7632 for (int i = 0; i < number_layers; ++i) {
7633 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7634 config.simulcast_layers[i].active = true;
7635 }
7636 config.video_stream_factory =
7637 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7638 "VP8", /*max qp*/ 56, /*screencast*/ false,
7639 /*screenshare enabled*/ false);
7640 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7641 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7642 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7643 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7644
7645 // First initialization.
7646 // Encoder should be initialized. Next frame should be key frame.
7647 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7648 sink_.SetNumExpectedLayers(number_layers);
7649 int64_t timestamp_ms = kFrameIntervalMs;
7650 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7651 WaitForEncodedFrame(timestamp_ms);
7652 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7653 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7654 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7655 VideoFrameType::kVideoFrameKey,
7656 VideoFrameType::kVideoFrameKey}));
7657
7658 // Disable top layer.
7659 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7660 config.simulcast_layers[number_layers - 1].active = false;
7661 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7662 sink_.SetNumExpectedLayers(number_layers - 1);
7663 timestamp_ms += kFrameIntervalMs;
7664 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7665 WaitForEncodedFrame(timestamp_ms);
7666 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7667 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7668 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7669 VideoFrameType::kVideoFrameDelta,
7670 VideoFrameType::kVideoFrameDelta}));
7671
7672 // Re-enable top layer.
7673 // Encoder should be re-initialized. Next frame should be key frame.
7674 config.simulcast_layers[number_layers - 1].active = true;
7675 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7676 sink_.SetNumExpectedLayers(number_layers);
7677 timestamp_ms += kFrameIntervalMs;
7678 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7679 WaitForEncodedFrame(timestamp_ms);
7680 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7681 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7682 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7683 VideoFrameType::kVideoFrameKey,
7684 VideoFrameType::kVideoFrameKey}));
7685
7686 // Top layer max rate change.
7687 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7688 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7689 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7690 sink_.SetNumExpectedLayers(number_layers);
7691 timestamp_ms += kFrameIntervalMs;
7692 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7693 WaitForEncodedFrame(timestamp_ms);
7694 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7695 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7696 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7697 VideoFrameType::kVideoFrameDelta,
7698 VideoFrameType::kVideoFrameDelta}));
7699
7700 // Top layer resolution change.
7701 // Encoder should be re-initialized. Next frame should be key frame.
7702 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7703 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7704 sink_.SetNumExpectedLayers(number_layers);
7705 timestamp_ms += kFrameIntervalMs;
7706 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7707 WaitForEncodedFrame(timestamp_ms);
7708 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7709 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7710 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7711 VideoFrameType::kVideoFrameKey,
7712 VideoFrameType::kVideoFrameKey}));
7713 video_stream_encoder_->Stop();
7714}
7715
Henrik Boström1124ed12021-02-25 10:30:39 +01007716TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7717 const int kFrameWidth = 1280;
7718 const int kFrameHeight = 720;
7719
7720 SetUp();
7721 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7722 DataRate::BitsPerSec(kTargetBitrateBps),
7723 DataRate::BitsPerSec(kTargetBitrateBps),
7724 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7725
7726 // Capturing a frame should reconfigure the encoder and expose the encoder
7727 // resolution, which is the same as the input frame.
7728 int64_t timestamp_ms = kFrameIntervalMs;
7729 video_source_.IncomingCapturedFrame(
7730 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7731 WaitForEncodedFrame(timestamp_ms);
7732 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7733 EXPECT_THAT(video_source_.sink_wants().resolutions,
7734 ::testing::ElementsAreArray(
7735 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7736
7737 video_stream_encoder_->Stop();
7738}
7739
7740TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7741 // Pick downscale factors such that we never encode at full resolution - this
7742 // is an interesting use case. The frame resolution influences the encoder
7743 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7744 // encoder should not ask for the frame resolution. This allows video frames
7745 // to have the appearence of one resolution but optimize its internal buffers
7746 // for what is actually encoded.
7747 const size_t kNumSimulcastLayers = 3u;
7748 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7749 const int kFrameWidth = 1280;
7750 const int kFrameHeight = 720;
7751 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7752 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7753 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7754 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7755 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7756 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7757
7758 VideoEncoderConfig config;
7759 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7760 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7761 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7762 config.simulcast_layers[i].active = true;
7763 }
7764 config.video_stream_factory =
7765 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7766 "VP8", /*max qp*/ 56, /*screencast*/ false,
7767 /*screenshare enabled*/ false);
7768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7769 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7770 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7771 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7772
7773 // Capture a frame with all layers active.
7774 int64_t timestamp_ms = kFrameIntervalMs;
7775 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7776 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7777 video_source_.IncomingCapturedFrame(
7778 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7779 WaitForEncodedFrame(timestamp_ms);
7780 // Expect encoded resolutions to match the expected simulcast layers.
7781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7782 EXPECT_THAT(
7783 video_source_.sink_wants().resolutions,
7784 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7785
7786 // Capture a frame with one of the layers inactive.
7787 timestamp_ms += kFrameIntervalMs;
7788 config.simulcast_layers[2].active = false;
7789 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7790 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7791 video_source_.IncomingCapturedFrame(
7792 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7793 WaitForEncodedFrame(timestamp_ms);
7794
7795 // Expect encoded resolutions to match the expected simulcast layers.
7796 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7797 EXPECT_THAT(video_source_.sink_wants().resolutions,
7798 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7799
7800 // Capture a frame with all but one layer turned off.
7801 timestamp_ms += kFrameIntervalMs;
7802 config.simulcast_layers[1].active = false;
7803 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7804 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7805 video_source_.IncomingCapturedFrame(
7806 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7807 WaitForEncodedFrame(timestamp_ms);
7808
7809 // Expect encoded resolutions to match the expected simulcast layers.
7810 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7811 EXPECT_THAT(video_source_.sink_wants().resolutions,
7812 ::testing::ElementsAreArray({kLayer0Size}));
7813
7814 video_stream_encoder_->Stop();
7815}
7816
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007817TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
7818 // Enable encoder source to force encoder reconfig.
7819 encoder_factory_.SetHasInternalSource(true);
7820 ResetEncoder("VP8", 1, 1, 1, false);
7821
7822 // Set QP on encoded frame and pass the frame to encode complete callback.
7823 // Since QP is present QP parsing won't be triggered and the original value
7824 // should be kept.
7825 EncodedImage encoded_image;
7826 encoded_image.qp_ = 123;
7827 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7828 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7829 CodecSpecificInfo codec_info;
7830 codec_info.codecType = kVideoCodecVP8;
7831 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7832 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7833 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7834 video_stream_encoder_->Stop();
7835}
7836
7837TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
7838 // Enable encoder source to force encoder reconfig.
7839 encoder_factory_.SetHasInternalSource(true);
7840 ResetEncoder("VP8", 1, 1, 1, false);
7841
7842 // Pass an encoded frame without QP to encode complete callback. QP should be
7843 // parsed and set.
7844 EncodedImage encoded_image;
7845 encoded_image.qp_ = -1;
7846 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7847 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7848 CodecSpecificInfo codec_info;
7849 codec_info.codecType = kVideoCodecVP8;
7850 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7851 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7852 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7853 video_stream_encoder_->Stop();
7854}
7855
7856TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7857 webrtc::test::ScopedFieldTrials field_trials(
7858 "WebRTC-QpParsingKillSwitch/Enabled/");
7859
7860 // Enable encoder source to force encoder reconfig.
7861 encoder_factory_.SetHasInternalSource(true);
7862 ResetEncoder("VP8", 1, 1, 1, false);
7863
7864 EncodedImage encoded_image;
7865 encoded_image.qp_ = -1;
7866 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7867 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7868 CodecSpecificInfo codec_info;
7869 codec_info.codecType = kVideoCodecVP8;
7870 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7871 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7872 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7873 video_stream_encoder_->Stop();
7874}
7875
perkj26091b12016-09-01 01:17:40 -07007876} // namespace webrtc