blob: a76820367e7ed017fd0bf0cf7ee6004355a3340d [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020040#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "system_wrappers/include/sleep.h"
42#include "test/encoder_settings.h"
43#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020044#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010045#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/gmock.h"
47#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020048#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020049#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070050
51namespace webrtc {
52
sprang57c2fff2017-01-16 06:24:02 -080053using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020054using ::testing::AllOf;
55using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010056using ::testing::Matcher;
57using ::testing::NiceMock;
58using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080060
perkj803d97f2016-11-01 11:45:46 -070061namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020062const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010063const int kQpLow = 1;
64const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int kMinFramerateFps = 2;
66const int kMinBalancedFramerateFps = 7;
67const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080068const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010069const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020070const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010071const uint32_t kSimulcastTargetBitrateBps = 3150000;
72const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080073const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070074const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020076const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020077const VideoEncoder::ResolutionBitrateLimits
78 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
79const VideoEncoder::ResolutionBitrateLimits
80 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080081
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020082uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
83 0x00, 0x00, 0x03, 0x03, 0xF4,
84 0x05, 0x03, 0xC7, 0xE0, 0x1B,
85 0x41, 0x10, 0x8D, 0x00};
86
perkj803d97f2016-11-01 11:45:46 -070087class TestBuffer : public webrtc::I420Buffer {
88 public:
89 TestBuffer(rtc::Event* event, int width, int height)
90 : I420Buffer(width, height), event_(event) {}
91
92 private:
93 friend class rtc::RefCountedObject<TestBuffer>;
94 ~TestBuffer() override {
95 if (event_)
96 event_->Set();
97 }
98 rtc::Event* const event_;
99};
100
Noah Richards51db4212019-06-12 06:59:12 -0700101// A fake native buffer that can't be converted to I420.
102class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
103 public:
104 FakeNativeBuffer(rtc::Event* event, int width, int height)
105 : event_(event), width_(width), height_(height) {}
106 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
107 int width() const override { return width_; }
108 int height() const override { return height_; }
109 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
110 return nullptr;
111 }
112
113 private:
114 friend class rtc::RefCountedObject<FakeNativeBuffer>;
115 ~FakeNativeBuffer() override {
116 if (event_)
117 event_->Set();
118 }
119 rtc::Event* const event_;
120 const int width_;
121 const int height_;
122};
123
Niels Möller7dc26b72017-12-06 10:27:48 +0100124class CpuOveruseDetectorProxy : public OveruseFrameDetector {
125 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200126 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
127 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200128 last_target_framerate_fps_(-1),
129 framerate_updated_event_(true /* manual_reset */,
130 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100131 virtual ~CpuOveruseDetectorProxy() {}
132
133 void OnTargetFramerateUpdated(int framerate_fps) override {
134 rtc::CritScope cs(&lock_);
135 last_target_framerate_fps_ = framerate_fps;
136 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200137 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100138 }
139
140 int GetLastTargetFramerate() {
141 rtc::CritScope cs(&lock_);
142 return last_target_framerate_fps_;
143 }
144
Niels Möller4db138e2018-04-19 09:04:13 +0200145 CpuOveruseOptions GetOptions() { return options_; }
146
Henrik Boström381d1092020-05-12 18:49:07 +0200147 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
148
Niels Möller7dc26b72017-12-06 10:27:48 +0100149 private:
150 rtc::CriticalSection lock_;
151 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200152 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100153};
154
Henrik Boström91aa7322020-04-28 12:24:33 +0200155class FakeQualityScalerQpUsageHandlerCallback
156 : public QualityScalerQpUsageHandlerCallbackInterface {
157 public:
158 FakeQualityScalerQpUsageHandlerCallback()
Henrik Boström5bf60e42020-05-13 15:20:25 +0200159 : QualityScalerQpUsageHandlerCallbackInterface(),
160 qp_usage_handled_event_(/*manual_reset=*/true,
161 /*initially_signaled=*/false),
162 clear_qp_samples_result_(absl::nullopt) {}
163 ~FakeQualityScalerQpUsageHandlerCallback() override {
164 RTC_DCHECK(clear_qp_samples_result_.has_value());
165 }
Henrik Boström91aa7322020-04-28 12:24:33 +0200166
167 void OnQpUsageHandled(bool clear_qp_samples) override {
168 clear_qp_samples_result_ = clear_qp_samples;
Henrik Boström5bf60e42020-05-13 15:20:25 +0200169 qp_usage_handled_event_.Set();
Henrik Boström91aa7322020-04-28 12:24:33 +0200170 }
171
Henrik Boström5bf60e42020-05-13 15:20:25 +0200172 bool WaitForQpUsageHandled() { return qp_usage_handled_event_.Wait(5000); }
173
Henrik Boström91aa7322020-04-28 12:24:33 +0200174 absl::optional<bool> clear_qp_samples_result() const {
175 return clear_qp_samples_result_;
176 }
177
178 private:
Henrik Boström5bf60e42020-05-13 15:20:25 +0200179 rtc::Event qp_usage_handled_event_;
Henrik Boström91aa7322020-04-28 12:24:33 +0200180 absl::optional<bool> clear_qp_samples_result_;
181};
182
Henrik Boström381d1092020-05-12 18:49:07 +0200183class VideoSourceRestrictionsUpdatedListener
184 : public ResourceAdaptationProcessorListener {
185 public:
186 VideoSourceRestrictionsUpdatedListener()
187 : was_restrictions_updated_(false), restrictions_updated_event_() {}
188 ~VideoSourceRestrictionsUpdatedListener() override {
189 RTC_DCHECK(was_restrictions_updated_);
190 }
191
192 rtc::Event* restrictions_updated_event() {
193 return &restrictions_updated_event_;
194 }
195
196 // ResourceAdaptationProcessorListener implementation.
197 void OnVideoSourceRestrictionsUpdated(
198 VideoSourceRestrictions restrictions,
199 const VideoAdaptationCounters& adaptation_counters,
200 rtc::scoped_refptr<Resource> reason) override {
201 was_restrictions_updated_ = true;
202 restrictions_updated_event_.Set();
203 }
204
205 private:
206 bool was_restrictions_updated_;
207 rtc::Event restrictions_updated_event_;
208};
209
mflodmancc3d4422017-08-03 08:27:51 -0700210class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700211 public:
Niels Möller213618e2018-07-24 09:29:58 +0200212 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200213 const VideoStreamEncoderSettings& settings,
214 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100215 : VideoStreamEncoder(Clock::GetRealTimeClock(),
216 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200217 stats_proxy,
218 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200219 std::unique_ptr<OveruseFrameDetector>(
220 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100221 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100222 task_queue_factory),
Henrik Boströmc55516d2020-05-11 16:29:22 +0200223 fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
224 fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
Henrik Boström381d1092020-05-12 18:49:07 +0200225 fake_cpu_resource_->Initialize(encoder_queue(),
226 resource_adaptation_queue());
227 fake_quality_resource_->Initialize(encoder_queue(),
228 resource_adaptation_queue());
Henrik Boströmc55516d2020-05-11 16:29:22 +0200229 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200230 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200231 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100232 }
perkj803d97f2016-11-01 11:45:46 -0700233
Henrik Boström381d1092020-05-12 18:49:07 +0200234 void SetSourceAndWaitForRestrictionsUpdated(
235 rtc::VideoSourceInterface<VideoFrame>* source,
236 const DegradationPreference& degradation_preference) {
237 VideoSourceRestrictionsUpdatedListener listener;
238 AddAdaptationListenerForTesting(&listener);
239 SetSource(source, degradation_preference);
240 listener.restrictions_updated_event()->Wait(5000);
241 RemoveAdaptationListenerForTesting(&listener);
242 }
243
244 void SetSourceAndWaitForFramerateUpdated(
245 rtc::VideoSourceInterface<VideoFrame>* source,
246 const DegradationPreference& degradation_preference) {
247 overuse_detector_proxy_->framerate_updated_event()->Reset();
248 SetSource(source, degradation_preference);
249 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
250 }
251
252 void OnBitrateUpdatedAndWaitForManagedResources(
253 DataRate target_bitrate,
254 DataRate stable_target_bitrate,
255 DataRate link_allocation,
256 uint8_t fraction_lost,
257 int64_t round_trip_time_ms,
258 double cwnd_reduce_ratio) {
259 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
260 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
261 // Bitrate is updated on the encoder queue.
262 WaitUntilTaskQueueIsIdle();
263 // Give the managed resources time to react to the new bitrate.
264 // TODO(hbos): Can we await an appropriate event instead?
265 WaitUntilAdaptationTaskQueueIsIdle();
266 }
267
268 void WaitUntilAdaptationTaskQueueIsIdle() {
269 rtc::Event event;
270 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
271 ASSERT_TRUE(event.Wait(5000));
272 }
273
kthelgason2fc52542017-03-03 00:24:41 -0800274 // This is used as a synchronisation mechanism, to make sure that the
275 // encoder queue is not blocked before we start sending it frames.
276 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100277 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200278 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800279 ASSERT_TRUE(event.Wait(5000));
280 }
281
Henrik Boström91aa7322020-04-28 12:24:33 +0200282 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200283 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200284 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200285 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200286 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
287 event.Set();
288 });
289 ASSERT_TRUE(event.Wait(5000));
290 }
291 void TriggerCpuUnderuse() {
292 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200293 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200294 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
295 event.Set();
296 });
297 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200298 }
kthelgason876222f2016-11-29 01:44:11 -0800299
Henrik Boström91aa7322020-04-28 12:24:33 +0200300 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200301 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200302 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200303 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200304 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
305 event.Set();
306 });
307 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200308 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200309 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200310 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200311 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200312 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
313 event.Set();
314 });
315 ASSERT_TRUE(event.Wait(5000));
316 }
317
318 // Fakes high QP resource usage measurements on the real
319 // QualityScalerResource. Returns whether or not QP samples would have been
320 // cleared if this had been a real signal from the QualityScaler.
321 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200322 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
323 new FakeQualityScalerQpUsageHandlerCallback();
Henrik Boström5bf60e42020-05-13 15:20:25 +0200324 encoder_queue()->PostTask([this, callback] {
325 // This will cause a "ping" between adaptation task queue and encoder
326 // queue. When we have the result, the |callback| will be notified.
Henrik Boström91aa7322020-04-28 12:24:33 +0200327 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
Henrik Boström91aa7322020-04-28 12:24:33 +0200328 });
Henrik Boström5bf60e42020-05-13 15:20:25 +0200329 EXPECT_TRUE(callback->WaitForQpUsageHandled());
Henrik Boström91aa7322020-04-28 12:24:33 +0200330 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
331 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200332 }
sprangfda496a2017-06-15 04:21:07 -0700333
Niels Möller7dc26b72017-12-06 10:27:48 +0100334 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200335 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
336 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700337};
338
asapersson5f7226f2016-11-25 04:37:00 -0800339class VideoStreamFactory
340 : public VideoEncoderConfig::VideoStreamFactoryInterface {
341 public:
sprangfda496a2017-06-15 04:21:07 -0700342 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
343 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800344 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700345 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800346 }
347
348 private:
349 std::vector<VideoStream> CreateEncoderStreams(
350 int width,
351 int height,
352 const VideoEncoderConfig& encoder_config) override {
353 std::vector<VideoStream> streams =
354 test::CreateVideoStreams(width, height, encoder_config);
355 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100356 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700357 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800358 }
359 return streams;
360 }
sprangfda496a2017-06-15 04:21:07 -0700361
asapersson5f7226f2016-11-25 04:37:00 -0800362 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700363 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800364};
365
Noah Richards51db4212019-06-12 06:59:12 -0700366// Simulates simulcast behavior and makes highest stream resolutions divisible
367// by 4.
368class CroppingVideoStreamFactory
369 : public VideoEncoderConfig::VideoStreamFactoryInterface {
370 public:
371 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
372 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
373 EXPECT_GT(num_temporal_layers, 0u);
374 EXPECT_GT(framerate, 0);
375 }
376
377 private:
378 std::vector<VideoStream> CreateEncoderStreams(
379 int width,
380 int height,
381 const VideoEncoderConfig& encoder_config) override {
382 std::vector<VideoStream> streams = test::CreateVideoStreams(
383 width - width % 4, height - height % 4, encoder_config);
384 for (VideoStream& stream : streams) {
385 stream.num_temporal_layers = num_temporal_layers_;
386 stream.max_framerate = framerate_;
387 }
388 return streams;
389 }
390
391 const size_t num_temporal_layers_;
392 const int framerate_;
393};
394
sprangb1ca0732017-02-01 08:38:12 -0800395class AdaptingFrameForwarder : public test::FrameForwarder {
396 public:
397 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700398 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800399
400 void set_adaptation_enabled(bool enabled) {
401 rtc::CritScope cs(&crit_);
402 adaptation_enabled_ = enabled;
403 }
404
asaperssonfab67072017-04-04 05:51:49 -0700405 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800406 rtc::CritScope cs(&crit_);
407 return adaptation_enabled_;
408 }
409
asapersson09f05612017-05-15 23:40:18 -0700410 rtc::VideoSinkWants last_wants() const {
411 rtc::CritScope cs(&crit_);
412 return last_wants_;
413 }
414
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200415 absl::optional<int> last_sent_width() const { return last_width_; }
416 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800417
sprangb1ca0732017-02-01 08:38:12 -0800418 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
419 int cropped_width = 0;
420 int cropped_height = 0;
421 int out_width = 0;
422 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700423 if (adaption_enabled()) {
424 if (adapter_.AdaptFrameResolution(
425 video_frame.width(), video_frame.height(),
426 video_frame.timestamp_us() * 1000, &cropped_width,
427 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100428 VideoFrame adapted_frame =
429 VideoFrame::Builder()
430 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
431 nullptr, out_width, out_height))
432 .set_timestamp_rtp(99)
433 .set_timestamp_ms(99)
434 .set_rotation(kVideoRotation_0)
435 .build();
sprangc5d62e22017-04-02 23:53:04 -0700436 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100437 if (video_frame.has_update_rect()) {
438 adapted_frame.set_update_rect(
439 video_frame.update_rect().ScaleWithFrame(
440 video_frame.width(), video_frame.height(), 0, 0,
441 video_frame.width(), video_frame.height(), out_width,
442 out_height));
443 }
sprangc5d62e22017-04-02 23:53:04 -0700444 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800445 last_width_.emplace(adapted_frame.width());
446 last_height_.emplace(adapted_frame.height());
447 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200448 last_width_ = absl::nullopt;
449 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700450 }
sprangb1ca0732017-02-01 08:38:12 -0800451 } else {
452 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800453 last_width_.emplace(video_frame.width());
454 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800455 }
456 }
457
458 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
459 const rtc::VideoSinkWants& wants) override {
460 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700461 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100462 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800463 test::FrameForwarder::AddOrUpdateSink(sink, wants);
464 }
sprangb1ca0732017-02-01 08:38:12 -0800465 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700466 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
467 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200468 absl::optional<int> last_width_;
469 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800470};
sprangc5d62e22017-04-02 23:53:04 -0700471
Niels Möller213618e2018-07-24 09:29:58 +0200472// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700473class MockableSendStatisticsProxy : public SendStatisticsProxy {
474 public:
475 MockableSendStatisticsProxy(Clock* clock,
476 const VideoSendStream::Config& config,
477 VideoEncoderConfig::ContentType content_type)
478 : SendStatisticsProxy(clock, config, content_type) {}
479
480 VideoSendStream::Stats GetStats() override {
481 rtc::CritScope cs(&lock_);
482 if (mock_stats_)
483 return *mock_stats_;
484 return SendStatisticsProxy::GetStats();
485 }
486
Niels Möller213618e2018-07-24 09:29:58 +0200487 int GetInputFrameRate() const override {
488 rtc::CritScope cs(&lock_);
489 if (mock_stats_)
490 return mock_stats_->input_frame_rate;
491 return SendStatisticsProxy::GetInputFrameRate();
492 }
sprangc5d62e22017-04-02 23:53:04 -0700493 void SetMockStats(const VideoSendStream::Stats& stats) {
494 rtc::CritScope cs(&lock_);
495 mock_stats_.emplace(stats);
496 }
497
498 void ResetMockStats() {
499 rtc::CritScope cs(&lock_);
500 mock_stats_.reset();
501 }
502
503 private:
504 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200505 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700506};
507
sprang4847ae62017-06-27 07:06:52 -0700508class MockBitrateObserver : public VideoBitrateAllocationObserver {
509 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200510 MOCK_METHOD(void,
511 OnBitrateAllocationUpdated,
512 (const VideoBitrateAllocation&),
513 (override));
sprang4847ae62017-06-27 07:06:52 -0700514};
515
philipel9b058032020-02-10 11:30:00 +0100516class MockEncoderSelector
517 : public VideoEncoderFactory::EncoderSelectorInterface {
518 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200519 MOCK_METHOD(void,
520 OnCurrentEncoder,
521 (const SdpVideoFormat& format),
522 (override));
523 MOCK_METHOD(absl::optional<SdpVideoFormat>,
524 OnAvailableBitrate,
525 (const DataRate& rate),
526 (override));
527 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100528};
529
perkj803d97f2016-11-01 11:45:46 -0700530} // namespace
531
mflodmancc3d4422017-08-03 08:27:51 -0700532class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700533 public:
534 static const int kDefaultTimeoutMs = 30 * 1000;
535
mflodmancc3d4422017-08-03 08:27:51 -0700536 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700537 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700538 codec_width_(320),
539 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200540 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200541 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700542 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200543 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700544 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700545 Clock::GetRealTimeClock(),
546 video_send_config_,
547 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700548 sink_(&fake_encoder_) {}
549
550 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700551 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700552 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200553 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800554 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200555 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200556 video_send_config_.rtp.payload_name = "FAKE";
557 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700558
Per512ecb32016-09-23 15:52:06 +0200559 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200560 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700561 video_encoder_config.video_stream_factory =
562 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100563 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700564
565 // Framerate limit is specified by the VideoStreamFactory.
566 std::vector<VideoStream> streams =
567 video_encoder_config.video_stream_factory->CreateEncoderStreams(
568 codec_width_, codec_height_, video_encoder_config);
569 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100570 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700571
Niels Möllerf1338562018-04-26 09:51:47 +0200572 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800573 }
574
Niels Möllerf1338562018-04-26 09:51:47 +0200575 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700576 if (video_stream_encoder_)
577 video_stream_encoder_->Stop();
578 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200579 stats_proxy_.get(), video_send_config_.encoder_settings,
580 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700581 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
582 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700583 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700584 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
585 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200586 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700587 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800588 }
589
590 void ResetEncoder(const std::string& payload_name,
591 size_t num_streams,
592 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700593 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700594 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200595 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800596
597 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200598 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800599 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100600 video_encoder_config.max_bitrate_bps =
601 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800602 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700603 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
604 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700605 video_encoder_config.content_type =
606 screenshare ? VideoEncoderConfig::ContentType::kScreen
607 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700608 if (payload_name == "VP9") {
609 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
610 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
611 video_encoder_config.encoder_specific_settings =
612 new rtc::RefCountedObject<
613 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
614 }
Niels Möllerf1338562018-04-26 09:51:47 +0200615 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700616 }
617
sprang57c2fff2017-01-16 06:24:02 -0800618 VideoFrame CreateFrame(int64_t ntp_time_ms,
619 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100620 VideoFrame frame =
621 VideoFrame::Builder()
622 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
623 destruction_event, codec_width_, codec_height_))
624 .set_timestamp_rtp(99)
625 .set_timestamp_ms(99)
626 .set_rotation(kVideoRotation_0)
627 .build();
sprang57c2fff2017-01-16 06:24:02 -0800628 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700629 return frame;
630 }
631
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100632 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
633 rtc::Event* destruction_event,
634 int offset_x) const {
635 VideoFrame frame =
636 VideoFrame::Builder()
637 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
638 destruction_event, codec_width_, codec_height_))
639 .set_timestamp_rtp(99)
640 .set_timestamp_ms(99)
641 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100642 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100643 .build();
644 frame.set_ntp_time_ms(ntp_time_ms);
645 return frame;
646 }
647
sprang57c2fff2017-01-16 06:24:02 -0800648 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100649 VideoFrame frame =
650 VideoFrame::Builder()
651 .set_video_frame_buffer(
652 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
653 .set_timestamp_rtp(99)
654 .set_timestamp_ms(99)
655 .set_rotation(kVideoRotation_0)
656 .build();
sprang57c2fff2017-01-16 06:24:02 -0800657 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700658 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700659 return frame;
660 }
661
Noah Richards51db4212019-06-12 06:59:12 -0700662 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
663 rtc::Event* destruction_event,
664 int width,
665 int height) const {
666 VideoFrame frame =
667 VideoFrame::Builder()
668 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
669 destruction_event, width, height))
670 .set_timestamp_rtp(99)
671 .set_timestamp_ms(99)
672 .set_rotation(kVideoRotation_0)
673 .build();
674 frame.set_ntp_time_ms(ntp_time_ms);
675 return frame;
676 }
677
678 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
679 rtc::Event* destruction_event) const {
680 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
681 codec_height_);
682 }
683
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100684 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
685 MockBitrateObserver bitrate_observer;
686 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
687
688 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
689 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100691 DataRate::BitsPerSec(kTargetBitrateBps),
692 DataRate::BitsPerSec(kTargetBitrateBps),
693 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100694
695 video_source_.IncomingCapturedFrame(
696 CreateFrame(1, codec_width_, codec_height_));
697 WaitForEncodedFrame(1);
698 }
699
asapersson02465b82017-04-10 01:12:52 -0700700 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700701 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700702 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
703 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700704 }
705
asapersson09f05612017-05-15 23:40:18 -0700706 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
707 const rtc::VideoSinkWants& wants2) {
708 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
709 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
710 }
711
Åsa Persson8c1bf952018-09-13 10:42:19 +0200712 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
713 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
714 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
715 EXPECT_FALSE(wants.target_pixel_count);
716 }
717
asapersson09f05612017-05-15 23:40:18 -0700718 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
719 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200720 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700721 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
722 EXPECT_GT(wants1.max_pixel_count, 0);
723 }
724
725 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
726 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200727 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700728 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
729 }
730
asaperssonf7e294d2017-06-13 23:25:22 -0700731 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
732 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200733 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700734 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
735 }
736
737 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
738 const rtc::VideoSinkWants& wants2) {
739 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
740 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
741 }
742
743 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
744 const rtc::VideoSinkWants& wants2) {
745 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
746 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
747 }
748
749 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
750 const rtc::VideoSinkWants& wants2) {
751 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
752 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
753 EXPECT_GT(wants1.max_pixel_count, 0);
754 }
755
756 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
757 const rtc::VideoSinkWants& wants2) {
758 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
759 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
760 }
761
asapersson09f05612017-05-15 23:40:18 -0700762 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
763 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200764 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700765 EXPECT_LT(wants.max_pixel_count, pixel_count);
766 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700767 }
768
769 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
770 EXPECT_LT(wants.max_framerate_fps, fps);
771 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
772 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700773 }
774
asaperssonf7e294d2017-06-13 23:25:22 -0700775 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
776 int expected_fps) {
777 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
778 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
779 EXPECT_FALSE(wants.target_pixel_count);
780 }
781
Jonathan Yubc771b72017-12-08 17:04:29 -0800782 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
783 int last_frame_pixels) {
784 // Balanced mode should always scale FPS to the desired range before
785 // attempting to scale resolution.
786 int fps_limit = wants.max_framerate_fps;
787 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100788 EXPECT_LE(7, fps_limit);
789 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800790 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100791 EXPECT_LE(10, fps_limit);
792 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800793 } else if (last_frame_pixels <= 640 * 480) {
794 EXPECT_LE(15, fps_limit);
795 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200796 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800797 }
798 }
799
sprang4847ae62017-06-27 07:06:52 -0700800 void WaitForEncodedFrame(int64_t expected_ntp_time) {
801 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100802 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700803 }
804
805 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
806 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100807 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700808 return ok;
809 }
810
811 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
812 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100813 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 }
815
816 void ExpectDroppedFrame() {
817 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100818 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700819 }
820
821 bool WaitForFrame(int64_t timeout_ms) {
822 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100823 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700824 return ok;
825 }
826
perkj26091b12016-09-01 01:17:40 -0700827 class TestEncoder : public test::FakeEncoder {
828 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100829 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700830
asaperssonfab67072017-04-04 05:51:49 -0700831 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800832 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700833 return config_;
834 }
835
836 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800837 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700838 block_next_encode_ = true;
839 }
840
Erik Språngaed30702018-11-05 12:57:17 +0100841 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800842 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100843 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100844 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100845 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100846 info.scaling_settings = VideoEncoder::ScalingSettings(
847 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100848 }
849 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100850 for (int i = 0; i < kMaxSpatialLayers; ++i) {
851 if (temporal_layers_supported_[i]) {
852 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
853 info.fps_allocation[i].resize(num_layers);
854 }
855 }
Erik Språngaed30702018-11-05 12:57:17 +0100856 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200857
858 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100859 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100860 return info;
kthelgason876222f2016-11-29 01:44:11 -0800861 }
862
Erik Språngb7cb7b52019-02-26 15:52:33 +0100863 int32_t RegisterEncodeCompleteCallback(
864 EncodedImageCallback* callback) override {
865 rtc::CritScope lock(&local_crit_sect_);
866 encoded_image_callback_ = callback;
867 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
868 }
869
perkjfa10b552016-10-02 23:45:26 -0700870 void ContinueEncode() { continue_encode_event_.Set(); }
871
872 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
873 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800874 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700875 EXPECT_EQ(timestamp_, timestamp);
876 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
877 }
878
kthelgason2fc52542017-03-03 00:24:41 -0800879 void SetQualityScaling(bool b) {
880 rtc::CritScope lock(&local_crit_sect_);
881 quality_scaling_ = b;
882 }
kthelgasonad9010c2017-02-14 00:46:51 -0800883
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100884 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
885 rtc::CritScope lock(&local_crit_sect_);
886 requested_resolution_alignment_ = requested_resolution_alignment;
887 }
888
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100889 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
890 rtc::CritScope lock(&local_crit_sect_);
891 is_hardware_accelerated_ = is_hardware_accelerated;
892 }
893
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100894 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
895 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
896 rtc::CritScope lock(&local_crit_sect_);
897 temporal_layers_supported_[spatial_idx] = supported;
898 }
899
Sergey Silkin6456e352019-07-08 17:56:40 +0200900 void SetResolutionBitrateLimits(
901 std::vector<ResolutionBitrateLimits> thresholds) {
902 rtc::CritScope cs(&local_crit_sect_);
903 resolution_bitrate_limits_ = thresholds;
904 }
905
sprangfe627f32017-03-29 08:24:59 -0700906 void ForceInitEncodeFailure(bool force_failure) {
907 rtc::CritScope lock(&local_crit_sect_);
908 force_init_encode_failed_ = force_failure;
909 }
910
Niels Möller6bb5ab92019-01-11 11:11:10 +0100911 void SimulateOvershoot(double rate_factor) {
912 rtc::CritScope lock(&local_crit_sect_);
913 rate_factor_ = rate_factor;
914 }
915
Erik Språngd7329ca2019-02-21 21:19:53 +0100916 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100917 rtc::CritScope lock(&local_crit_sect_);
918 return last_framerate_;
919 }
920
Erik Språngd7329ca2019-02-21 21:19:53 +0100921 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100922 rtc::CritScope lock(&local_crit_sect_);
923 return last_update_rect_;
924 }
925
Niels Möller87e2d782019-03-07 10:18:23 +0100926 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100927 rtc::CritScope lock(&local_crit_sect_);
928 return last_frame_types_;
929 }
930
931 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100932 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100933 keyframe ? VideoFrameType::kVideoFrameKey
934 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100935 {
936 rtc::CritScope lock(&local_crit_sect_);
937 last_frame_types_ = frame_type;
938 }
Niels Möllerb859b322019-03-07 12:40:01 +0100939 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100940 }
941
Erik Språngb7cb7b52019-02-26 15:52:33 +0100942 void InjectEncodedImage(const EncodedImage& image) {
943 rtc::CritScope lock(&local_crit_sect_);
944 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
945 }
946
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200947 void InjectEncodedImage(const EncodedImage& image,
948 const CodecSpecificInfo* codec_specific_info,
949 const RTPFragmentationHeader* fragmentation) {
950 rtc::CritScope lock(&local_crit_sect_);
951 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
952 fragmentation);
953 }
954
Erik Språngd7329ca2019-02-21 21:19:53 +0100955 void ExpectNullFrame() {
956 rtc::CritScope lock(&local_crit_sect_);
957 expect_null_frame_ = true;
958 }
959
Erik Språng5056af02019-09-02 15:53:11 +0200960 absl::optional<VideoEncoder::RateControlParameters>
961 GetAndResetLastRateControlSettings() {
962 auto settings = last_rate_control_settings_;
963 last_rate_control_settings_.reset();
964 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100965 }
966
Sergey Silkin5ee69672019-07-02 14:18:34 +0200967 int GetNumEncoderInitializations() const {
968 rtc::CritScope lock(&local_crit_sect_);
969 return num_encoder_initializations_;
970 }
971
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200972 int GetNumSetRates() const {
973 rtc::CritScope lock(&local_crit_sect_);
974 return num_set_rates_;
975 }
976
perkjfa10b552016-10-02 23:45:26 -0700977 private:
perkj26091b12016-09-01 01:17:40 -0700978 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100979 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700980 bool block_encode;
981 {
brandtre78d2662017-01-16 05:57:16 -0800982 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100983 if (expect_null_frame_) {
984 EXPECT_EQ(input_image.timestamp(), 0u);
985 EXPECT_EQ(input_image.width(), 1);
986 last_frame_types_ = *frame_types;
987 expect_null_frame_ = false;
988 } else {
989 EXPECT_GT(input_image.timestamp(), timestamp_);
990 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
991 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
992 }
perkj26091b12016-09-01 01:17:40 -0700993
994 timestamp_ = input_image.timestamp();
995 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700996 last_input_width_ = input_image.width();
997 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700998 block_encode = block_next_encode_;
999 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001000 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001001 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -07001002 }
Niels Möllerb859b322019-03-07 12:40:01 +01001003 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001004 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001005 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001006 return result;
1007 }
1008
sprangfe627f32017-03-29 08:24:59 -07001009 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001010 const Settings& settings) override {
1011 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001012
sprangfe627f32017-03-29 08:24:59 -07001013 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001014 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001015
1016 ++num_encoder_initializations_;
1017
Erik Språng82fad3d2018-03-21 09:57:23 +01001018 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001019 // Simulate setting up temporal layers, in order to validate the life
1020 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001021 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001022 frame_buffer_controller_ =
1023 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001024 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001025 if (force_init_encode_failed_) {
1026 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001027 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001028 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001029
Erik Språngb7cb7b52019-02-26 15:52:33 +01001030 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001031 return res;
1032 }
1033
Erik Språngb7cb7b52019-02-26 15:52:33 +01001034 int32_t Release() override {
1035 rtc::CritScope lock(&local_crit_sect_);
1036 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1037 initialized_ = EncoderState::kUninitialized;
1038 return FakeEncoder::Release();
1039 }
1040
Erik Språng16cb8f52019-04-12 13:59:09 +02001041 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001042 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001043 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001044 VideoBitrateAllocation adjusted_rate_allocation;
1045 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1046 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001047 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001048 adjusted_rate_allocation.SetBitrate(
1049 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001050 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001051 rate_factor_));
1052 }
1053 }
1054 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001055 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001056 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001057 RateControlParameters adjusted_paramters = parameters;
1058 adjusted_paramters.bitrate = adjusted_rate_allocation;
1059 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001060 }
1061
brandtre78d2662017-01-16 05:57:16 -08001062 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001063 enum class EncoderState {
1064 kUninitialized,
1065 kInitializationFailed,
1066 kInitialized
1067 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
1068 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -07001069 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -07001070 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -07001071 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1072 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1073 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1074 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1075 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001076 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001077 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +01001078 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -07001079 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001080 absl::optional<bool>
1081 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
1082 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -07001083 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001084 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
1085 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001086 absl::optional<VideoEncoder::RateControlParameters>
1087 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001088 VideoFrame::UpdateRect last_update_rect_
1089 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001090 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001091 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001092 EncodedImageCallback* encoded_image_callback_
1093 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001094 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001095 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001096 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1097 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001098 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001099 };
1100
mflodmancc3d4422017-08-03 08:27:51 -07001101 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001102 public:
1103 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001104 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001105
perkj26091b12016-09-01 01:17:40 -07001106 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001107 EXPECT_TRUE(
1108 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1109 }
1110
1111 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1112 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001113 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001114 if (!encoded_frame_event_.Wait(timeout_ms))
1115 return false;
perkj26091b12016-09-01 01:17:40 -07001116 {
1117 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001118 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001119 }
1120 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001121 return true;
perkj26091b12016-09-01 01:17:40 -07001122 }
1123
sprangb1ca0732017-02-01 08:38:12 -08001124 void WaitForEncodedFrame(uint32_t expected_width,
1125 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001126 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001127 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001128 }
1129
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001130 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001131 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001132 uint32_t width = 0;
1133 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001134 {
1135 rtc::CritScope lock(&crit_);
1136 width = last_width_;
1137 height = last_height_;
1138 }
1139 EXPECT_EQ(expected_height, height);
1140 EXPECT_EQ(expected_width, width);
1141 }
1142
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001143 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1144 int width = 0;
1145 int height = 0;
1146 {
1147 rtc::CritScope lock(&crit_);
1148 width = last_width_;
1149 height = last_height_;
1150 }
1151 EXPECT_EQ(width % resolution_alignment, 0);
1152 EXPECT_EQ(height % resolution_alignment, 0);
1153 }
1154
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001155 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1156 VideoRotation rotation;
1157 {
1158 rtc::CritScope lock(&crit_);
1159 rotation = last_rotation_;
1160 }
1161 EXPECT_EQ(expected_rotation, rotation);
1162 }
1163
kthelgason2fc52542017-03-03 00:24:41 -08001164 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001165
sprangc5d62e22017-04-02 23:53:04 -07001166 bool WaitForFrame(int64_t timeout_ms) {
1167 return encoded_frame_event_.Wait(timeout_ms);
1168 }
1169
perkj26091b12016-09-01 01:17:40 -07001170 void SetExpectNoFrames() {
1171 rtc::CritScope lock(&crit_);
1172 expect_frames_ = false;
1173 }
1174
asaperssonfab67072017-04-04 05:51:49 -07001175 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001176 rtc::CritScope lock(&crit_);
1177 return number_of_reconfigurations_;
1178 }
1179
asaperssonfab67072017-04-04 05:51:49 -07001180 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001181 rtc::CritScope lock(&crit_);
1182 return min_transmit_bitrate_bps_;
1183 }
1184
Erik Språngd7329ca2019-02-21 21:19:53 +01001185 void SetNumExpectedLayers(size_t num_layers) {
1186 rtc::CritScope lock(&crit_);
1187 num_expected_layers_ = num_layers;
1188 }
1189
Erik Språngb7cb7b52019-02-26 15:52:33 +01001190 int64_t GetLastCaptureTimeMs() const {
1191 rtc::CritScope lock(&crit_);
1192 return last_capture_time_ms_;
1193 }
1194
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001195 std::vector<uint8_t> GetLastEncodedImageData() {
1196 rtc::CritScope lock(&crit_);
1197 return std::move(last_encoded_image_data_);
1198 }
1199
1200 RTPFragmentationHeader GetLastFragmentation() {
1201 rtc::CritScope lock(&crit_);
1202 return std::move(last_fragmentation_);
1203 }
1204
perkj26091b12016-09-01 01:17:40 -07001205 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001206 Result OnEncodedImage(
1207 const EncodedImage& encoded_image,
1208 const CodecSpecificInfo* codec_specific_info,
1209 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001210 rtc::CritScope lock(&crit_);
1211 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001212 last_encoded_image_data_ = std::vector<uint8_t>(
1213 encoded_image.data(), encoded_image.data() + encoded_image.size());
1214 if (fragmentation) {
1215 last_fragmentation_.CopyFrom(*fragmentation);
1216 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001217 uint32_t timestamp = encoded_image.Timestamp();
1218 if (last_timestamp_ != timestamp) {
1219 num_received_layers_ = 1;
1220 } else {
1221 ++num_received_layers_;
1222 }
1223 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001224 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001225 last_width_ = encoded_image._encodedWidth;
1226 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001227 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001228 if (num_received_layers_ == num_expected_layers_) {
1229 encoded_frame_event_.Set();
1230 }
sprangb1ca0732017-02-01 08:38:12 -08001231 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001232 }
1233
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001234 void OnEncoderConfigurationChanged(
1235 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001236 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001237 VideoEncoderConfig::ContentType content_type,
1238 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001239 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001240 ++number_of_reconfigurations_;
1241 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1242 }
1243
perkj26091b12016-09-01 01:17:40 -07001244 rtc::CriticalSection crit_;
1245 TestEncoder* test_encoder_;
1246 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001247 std::vector<uint8_t> last_encoded_image_data_;
1248 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001249 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001250 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001251 uint32_t last_height_ = 0;
1252 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001253 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001254 size_t num_expected_layers_ = 1;
1255 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001256 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001257 int number_of_reconfigurations_ = 0;
1258 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001259 };
1260
Sergey Silkin5ee69672019-07-02 14:18:34 +02001261 class VideoBitrateAllocatorProxyFactory
1262 : public VideoBitrateAllocatorFactory {
1263 public:
1264 VideoBitrateAllocatorProxyFactory()
1265 : bitrate_allocator_factory_(
1266 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1267
1268 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1269 const VideoCodec& codec) override {
1270 rtc::CritScope lock(&crit_);
1271 codec_config_ = codec;
1272 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1273 }
1274
1275 VideoCodec codec_config() const {
1276 rtc::CritScope lock(&crit_);
1277 return codec_config_;
1278 }
1279
1280 private:
1281 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1282
1283 rtc::CriticalSection crit_;
1284 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1285 };
1286
perkj26091b12016-09-01 01:17:40 -07001287 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001288 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001289 int codec_width_;
1290 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001291 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001292 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001293 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001294 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001295 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001296 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001297 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001298 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001299 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001300 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001301};
1302
mflodmancc3d4422017-08-03 08:27:51 -07001303TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001305 DataRate::BitsPerSec(kTargetBitrateBps),
1306 DataRate::BitsPerSec(kTargetBitrateBps),
1307 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001308 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001309 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001310 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001311 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001312 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001313}
1314
mflodmancc3d4422017-08-03 08:27:51 -07001315TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001316 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001317 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001318 // The encoder will cache up to one frame for a short duration. Adding two
1319 // frames means that the first frame will be dropped and the second frame will
1320 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001321 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001322 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001323 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001324
Henrik Boström381d1092020-05-12 18:49:07 +02001325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001326 DataRate::BitsPerSec(kTargetBitrateBps),
1327 DataRate::BitsPerSec(kTargetBitrateBps),
1328 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001329
Sebastian Janssona3177052018-04-10 13:05:49 +02001330 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001331 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001332 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1333
1334 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001335 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001336}
1337
mflodmancc3d4422017-08-03 08:27:51 -07001338TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001339 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001340 DataRate::BitsPerSec(kTargetBitrateBps),
1341 DataRate::BitsPerSec(kTargetBitrateBps),
1342 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001343 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001344 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001345
Henrik Boström381d1092020-05-12 18:49:07 +02001346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1347 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1348 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001349 // The encoder will cache up to one frame for a short duration. Adding two
1350 // frames means that the first frame will be dropped and the second frame will
1351 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001352 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001353 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001354
Henrik Boström381d1092020-05-12 18:49:07 +02001355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001356 DataRate::BitsPerSec(kTargetBitrateBps),
1357 DataRate::BitsPerSec(kTargetBitrateBps),
1358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001359 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001360 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1361 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001362 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001363}
1364
mflodmancc3d4422017-08-03 08:27:51 -07001365TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001366 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001367 DataRate::BitsPerSec(kTargetBitrateBps),
1368 DataRate::BitsPerSec(kTargetBitrateBps),
1369 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001370 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001371 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001372
1373 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001374 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001375
perkja49cbd32016-09-16 07:53:41 -07001376 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001377 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001378 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001379}
1380
mflodmancc3d4422017-08-03 08:27:51 -07001381TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001382 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001383 DataRate::BitsPerSec(kTargetBitrateBps),
1384 DataRate::BitsPerSec(kTargetBitrateBps),
1385 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001386
perkja49cbd32016-09-16 07:53:41 -07001387 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001388 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001389
mflodmancc3d4422017-08-03 08:27:51 -07001390 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001391 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001392 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001393 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1394 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001395}
1396
mflodmancc3d4422017-08-03 08:27:51 -07001397TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001399 DataRate::BitsPerSec(kTargetBitrateBps),
1400 DataRate::BitsPerSec(kTargetBitrateBps),
1401 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001402
1403 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001404 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001405 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001406 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1407 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001408 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1409 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001410 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001411 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001412
mflodmancc3d4422017-08-03 08:27:51 -07001413 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001414}
1415
Noah Richards51db4212019-06-12 06:59:12 -07001416TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001418 DataRate::BitsPerSec(kTargetBitrateBps),
1419 DataRate::BitsPerSec(kTargetBitrateBps),
1420 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001421
1422 rtc::Event frame_destroyed_event;
1423 video_source_.IncomingCapturedFrame(
1424 CreateFakeNativeFrame(1, &frame_destroyed_event));
1425 ExpectDroppedFrame();
1426 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1427 video_stream_encoder_->Stop();
1428}
1429
1430TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1431 // Use the cropping factory.
1432 video_encoder_config_.video_stream_factory =
1433 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1434 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1435 kMaxPayloadLength);
1436 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1437
1438 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001440 DataRate::BitsPerSec(kTargetBitrateBps),
1441 DataRate::BitsPerSec(kTargetBitrateBps),
1442 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001443 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1444 WaitForEncodedFrame(1);
1445 // The encoder will have been configured once.
1446 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1447 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1448 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1449
1450 // Now send in a fake frame that needs to be cropped as the width/height
1451 // aren't divisible by 4 (see CreateEncoderStreams above).
1452 rtc::Event frame_destroyed_event;
1453 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1454 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1455 ExpectDroppedFrame();
1456 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1457 video_stream_encoder_->Stop();
1458}
1459
Ying Wang9b881ab2020-02-07 14:29:32 +01001460TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001461 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001462 DataRate::BitsPerSec(kTargetBitrateBps),
1463 DataRate::BitsPerSec(kTargetBitrateBps),
1464 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001465 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1466 WaitForEncodedFrame(1);
1467
Henrik Boström381d1092020-05-12 18:49:07 +02001468 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001469 DataRate::BitsPerSec(kTargetBitrateBps),
1470 DataRate::BitsPerSec(kTargetBitrateBps),
1471 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001472 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1473 // frames. Adding two frames means that the first frame will be dropped and
1474 // the second frame will be sent to the encoder.
1475 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1476 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1477 WaitForEncodedFrame(3);
1478 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1479 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1480 WaitForEncodedFrame(5);
1481 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1482 video_stream_encoder_->Stop();
1483}
1484
mflodmancc3d4422017-08-03 08:27:51 -07001485TEST_F(VideoStreamEncoderTest,
1486 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001488 DataRate::BitsPerSec(kTargetBitrateBps),
1489 DataRate::BitsPerSec(kTargetBitrateBps),
1490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001491 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001492
1493 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001494 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001495 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001496 // The encoder will have been configured once when the first frame is
1497 // received.
1498 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001499
1500 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001501 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001502 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001503 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001504 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001505
1506 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001507 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001508 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001509 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001510 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001511
mflodmancc3d4422017-08-03 08:27:51 -07001512 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001513}
1514
mflodmancc3d4422017-08-03 08:27:51 -07001515TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001517 DataRate::BitsPerSec(kTargetBitrateBps),
1518 DataRate::BitsPerSec(kTargetBitrateBps),
1519 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001520
1521 // Capture a frame and wait for it to synchronize with the encoder thread.
1522 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001523 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001524 // The encoder will have been configured once.
1525 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001526 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1527 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1528
1529 codec_width_ *= 2;
1530 codec_height_ *= 2;
1531 // Capture a frame with a higher resolution and wait for it to synchronize
1532 // with the encoder thread.
1533 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001534 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001535 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1536 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001537 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001538
mflodmancc3d4422017-08-03 08:27:51 -07001539 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001540}
1541
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001542TEST_F(VideoStreamEncoderTest,
1543 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001545 DataRate::BitsPerSec(kTargetBitrateBps),
1546 DataRate::BitsPerSec(kTargetBitrateBps),
1547 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001548
1549 // Capture a frame and wait for it to synchronize with the encoder thread.
1550 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1551 WaitForEncodedFrame(1);
1552
1553 VideoEncoderConfig video_encoder_config;
1554 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1555 // Changing the max payload data length recreates encoder.
1556 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1557 kMaxPayloadLength / 2);
1558
1559 // Capture a frame and wait for it to synchronize with the encoder thread.
1560 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1561 WaitForEncodedFrame(2);
1562 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1563
1564 video_stream_encoder_->Stop();
1565}
1566
Sergey Silkin5ee69672019-07-02 14:18:34 +02001567TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001569 DataRate::BitsPerSec(kTargetBitrateBps),
1570 DataRate::BitsPerSec(kTargetBitrateBps),
1571 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001572
1573 VideoEncoderConfig video_encoder_config;
1574 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1575 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1576 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1577 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1578 kMaxPayloadLength);
1579
1580 // Capture a frame and wait for it to synchronize with the encoder thread.
1581 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1582 WaitForEncodedFrame(1);
1583 // The encoder will have been configured once when the first frame is
1584 // received.
1585 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1586 EXPECT_EQ(kTargetBitrateBps,
1587 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1588 EXPECT_EQ(kStartBitrateBps,
1589 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1590
Sergey Silkin6456e352019-07-08 17:56:40 +02001591 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1592 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001593 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1594 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1595 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1596 kMaxPayloadLength);
1597
1598 // Capture a frame and wait for it to synchronize with the encoder thread.
1599 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1600 WaitForEncodedFrame(2);
1601 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1602 // Bitrate limits have changed - rate allocator should be reconfigured,
1603 // encoder should not be reconfigured.
1604 EXPECT_EQ(kTargetBitrateBps * 2,
1605 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1606 EXPECT_EQ(kStartBitrateBps * 2,
1607 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1608 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1609
1610 video_stream_encoder_->Stop();
1611}
1612
Sergey Silkin6456e352019-07-08 17:56:40 +02001613TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001614 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001616 DataRate::BitsPerSec(kTargetBitrateBps),
1617 DataRate::BitsPerSec(kTargetBitrateBps),
1618 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001619
Sergey Silkincd02eba2020-01-20 14:48:40 +01001620 const uint32_t kMinEncBitrateKbps = 100;
1621 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001622 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001623 /*frame_size_pixels=*/codec_width_ * codec_height_,
1624 /*min_start_bitrate_bps=*/0,
1625 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1626 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001627 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1628
Sergey Silkincd02eba2020-01-20 14:48:40 +01001629 VideoEncoderConfig video_encoder_config;
1630 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1631 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1632 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1633 (kMinEncBitrateKbps + 1) * 1000;
1634 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1635 kMaxPayloadLength);
1636
1637 // When both encoder and app provide bitrate limits, the intersection of
1638 // provided sets should be used.
1639 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1640 WaitForEncodedFrame(1);
1641 EXPECT_EQ(kMaxEncBitrateKbps,
1642 bitrate_allocator_factory_.codec_config().maxBitrate);
1643 EXPECT_EQ(kMinEncBitrateKbps + 1,
1644 bitrate_allocator_factory_.codec_config().minBitrate);
1645
1646 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1647 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1648 (kMinEncBitrateKbps - 1) * 1000;
1649 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1650 kMaxPayloadLength);
1651 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001652 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001653 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001654 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001655 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001656 bitrate_allocator_factory_.codec_config().minBitrate);
1657
Sergey Silkincd02eba2020-01-20 14:48:40 +01001658 video_stream_encoder_->Stop();
1659}
1660
1661TEST_F(VideoStreamEncoderTest,
1662 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001664 DataRate::BitsPerSec(kTargetBitrateBps),
1665 DataRate::BitsPerSec(kTargetBitrateBps),
1666 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001667
1668 const uint32_t kMinAppBitrateKbps = 100;
1669 const uint32_t kMaxAppBitrateKbps = 200;
1670 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1671 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1672 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1673 /*frame_size_pixels=*/codec_width_ * codec_height_,
1674 /*min_start_bitrate_bps=*/0,
1675 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1676 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1677 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1678
1679 VideoEncoderConfig video_encoder_config;
1680 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1681 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1682 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1683 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001684 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1685 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001686
Sergey Silkincd02eba2020-01-20 14:48:40 +01001687 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1688 WaitForEncodedFrame(1);
1689 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001690 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001691 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001692 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001693
1694 video_stream_encoder_->Stop();
1695}
1696
1697TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001698 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001700 DataRate::BitsPerSec(kTargetBitrateBps),
1701 DataRate::BitsPerSec(kTargetBitrateBps),
1702 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001703
1704 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001705 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001706 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001707 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001708 fake_encoder_.SetResolutionBitrateLimits(
1709 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1710
1711 VideoEncoderConfig video_encoder_config;
1712 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1713 video_encoder_config.max_bitrate_bps = 0;
1714 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1715 kMaxPayloadLength);
1716
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001717 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001718 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1719 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001720 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1721 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001722 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1723 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1724
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001725 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001726 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1727 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001728 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1729 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001730 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1731 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1732
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001733 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001734 // encoder for 360p should be used.
1735 video_source_.IncomingCapturedFrame(
1736 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1737 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001738 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1739 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001740 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1741 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1742
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001743 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001744 // ignored.
1745 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1746 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001747 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1748 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001749 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1750 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001751 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1752 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001753 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1754 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1755
1756 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1757 // for 270p should be used.
1758 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1759 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001760 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1761 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001762 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1763 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1764
1765 video_stream_encoder_->Stop();
1766}
1767
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001768TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001770 DataRate::BitsPerSec(kTargetBitrateBps),
1771 DataRate::BitsPerSec(kTargetBitrateBps),
1772 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001773
1774 VideoEncoderConfig video_encoder_config;
1775 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1776 video_encoder_config.max_bitrate_bps = 0;
1777 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1778 kMaxPayloadLength);
1779
1780 // Encode 720p frame to get the default encoder target bitrate.
1781 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1782 WaitForEncodedFrame(1);
1783 const uint32_t kDefaultTargetBitrateFor720pKbps =
1784 bitrate_allocator_factory_.codec_config()
1785 .simulcastStream[0]
1786 .targetBitrate;
1787
1788 // Set the max recommended encoder bitrate to something lower than the default
1789 // target bitrate.
1790 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1791 1280 * 720, 10 * 1000, 10 * 1000,
1792 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1793 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1794
1795 // Change resolution to trigger encoder reinitialization.
1796 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1797 WaitForEncodedFrame(2);
1798 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1799 WaitForEncodedFrame(3);
1800
1801 // Ensure the target bitrate is capped by the max bitrate.
1802 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1803 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1804 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1805 .simulcastStream[0]
1806 .targetBitrate *
1807 1000,
1808 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1809
1810 video_stream_encoder_->Stop();
1811}
1812
mflodmancc3d4422017-08-03 08:27:51 -07001813TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001814 EXPECT_TRUE(video_source_.has_sinks());
1815 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001816 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001817 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001818 EXPECT_FALSE(video_source_.has_sinks());
1819 EXPECT_TRUE(new_video_source.has_sinks());
1820
mflodmancc3d4422017-08-03 08:27:51 -07001821 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001822}
1823
mflodmancc3d4422017-08-03 08:27:51 -07001824TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001825 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001826 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001827 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001828 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001829}
1830
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001831TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1832 constexpr int kRequestedResolutionAlignment = 7;
1833 video_source_.set_adaptation_enabled(true);
1834 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001836 DataRate::BitsPerSec(kTargetBitrateBps),
1837 DataRate::BitsPerSec(kTargetBitrateBps),
1838 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001839
1840 // On the 1st frame, we should have initialized the encoder and
1841 // asked for its resolution requirements.
1842 video_source_.IncomingCapturedFrame(
1843 CreateFrame(1, codec_width_, codec_height_));
1844 WaitForEncodedFrame(1);
1845 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1846 kRequestedResolutionAlignment);
1847
1848 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1849 // (It's up the to the encoder to potentially drop the previous frame,
1850 // to avoid coding back-to-back keyframes.)
1851 video_source_.IncomingCapturedFrame(
1852 CreateFrame(2, codec_width_, codec_height_));
1853 WaitForEncodedFrame(2);
1854 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1855
1856 video_stream_encoder_->Stop();
1857}
1858
Jonathan Yubc771b72017-12-08 17:04:29 -08001859TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1860 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001861 const int kWidth = 1280;
1862 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001863
1864 // We rely on the automatic resolution adaptation, but we handle framerate
1865 // adaptation manually by mocking the stats proxy.
1866 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001867
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001868 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001870 DataRate::BitsPerSec(kTargetBitrateBps),
1871 DataRate::BitsPerSec(kTargetBitrateBps),
1872 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001873 video_stream_encoder_->SetSource(&video_source_,
1874 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001875 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001876 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001877 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001878 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1879
Jonathan Yubc771b72017-12-08 17:04:29 -08001880 // Adapt down as far as possible.
1881 rtc::VideoSinkWants last_wants;
1882 int64_t t = 1;
1883 int loop_count = 0;
1884 do {
1885 ++loop_count;
1886 last_wants = video_source_.sink_wants();
1887
1888 // Simulate the framerate we've been asked to adapt to.
1889 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1890 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1891 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1892 mock_stats.input_frame_rate = fps;
1893 stats_proxy_->SetMockStats(mock_stats);
1894
1895 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1896 sink_.WaitForEncodedFrame(t);
1897 t += frame_interval_ms;
1898
mflodmancc3d4422017-08-03 08:27:51 -07001899 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001900 VerifyBalancedModeFpsRange(
1901 video_source_.sink_wants(),
1902 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1903 } while (video_source_.sink_wants().max_pixel_count <
1904 last_wants.max_pixel_count ||
1905 video_source_.sink_wants().max_framerate_fps <
1906 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001907
Jonathan Yubc771b72017-12-08 17:04:29 -08001908 // Verify that we've adapted all the way down.
1909 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001910 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001911 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1912 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001913 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001914 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1915 *video_source_.last_sent_height());
1916 EXPECT_EQ(kMinBalancedFramerateFps,
1917 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001918
Jonathan Yubc771b72017-12-08 17:04:29 -08001919 // Adapt back up the same number of times we adapted down.
1920 for (int i = 0; i < loop_count - 1; ++i) {
1921 last_wants = video_source_.sink_wants();
1922
1923 // Simulate the framerate we've been asked to adapt to.
1924 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1925 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1926 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1927 mock_stats.input_frame_rate = fps;
1928 stats_proxy_->SetMockStats(mock_stats);
1929
1930 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1931 sink_.WaitForEncodedFrame(t);
1932 t += frame_interval_ms;
1933
Henrik Boström91aa7322020-04-28 12:24:33 +02001934 video_stream_encoder_->TriggerCpuUnderuse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001935 VerifyBalancedModeFpsRange(
1936 video_source_.sink_wants(),
1937 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1938 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1939 last_wants.max_pixel_count ||
1940 video_source_.sink_wants().max_framerate_fps >
1941 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001942 }
1943
Åsa Persson8c1bf952018-09-13 10:42:19 +02001944 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001945 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001946 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001947 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1948 EXPECT_EQ((loop_count - 1) * 2,
1949 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001950
mflodmancc3d4422017-08-03 08:27:51 -07001951 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001952}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001953
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001954TEST_F(VideoStreamEncoderTest,
1955 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1956 video_stream_encoder_->OnBitrateUpdated(
1957 DataRate::BitsPerSec(kTargetBitrateBps),
1958 DataRate::BitsPerSec(kTargetBitrateBps),
1959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1960 VerifyNoLimitation(video_source_.sink_wants());
1961
1962 const int kFrameWidth = 1280;
1963 const int kFrameHeight = 720;
1964
1965 int64_t ntp_time = kFrameIntervalMs;
1966
1967 // Force an input frame rate to be available, or the adaptation call won't
1968 // know what framerate to adapt form.
1969 const int kInputFps = 30;
1970 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1971 stats.input_frame_rate = kInputFps;
1972 stats_proxy_->SetMockStats(stats);
1973
1974 video_source_.set_adaptation_enabled(true);
1975 video_stream_encoder_->SetSource(
1976 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
1977 VerifyNoLimitation(video_source_.sink_wants());
1978 video_source_.IncomingCapturedFrame(
1979 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1980 sink_.WaitForEncodedFrame(ntp_time);
1981 ntp_time += kFrameIntervalMs;
1982
1983 // Trigger CPU overuse.
1984 video_stream_encoder_->TriggerCpuOveruse();
1985 video_source_.IncomingCapturedFrame(
1986 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1987 sink_.WaitForEncodedFrame(ntp_time);
1988 ntp_time += kFrameIntervalMs;
1989
1990 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1991 EXPECT_EQ(std::numeric_limits<int>::max(),
1992 video_source_.sink_wants().max_pixel_count);
1993 // Some framerate constraint should be set.
1994 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1995 EXPECT_LT(restricted_fps, kInputFps);
1996 video_source_.IncomingCapturedFrame(
1997 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1998 sink_.WaitForEncodedFrame(ntp_time);
1999 ntp_time += 100;
2000
2001 video_stream_encoder_->SetSource(
2002 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2003 // Give the encoder queue time to process the change in degradation preference
2004 // by waiting for an encoded frame.
2005 video_source_.IncomingCapturedFrame(
2006 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2007 sink_.WaitForEncodedFrame(ntp_time);
2008 ntp_time += kFrameIntervalMs;
2009
2010 video_stream_encoder_->TriggerQualityLow();
2011 video_source_.IncomingCapturedFrame(
2012 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2013 sink_.WaitForEncodedFrame(ntp_time);
2014 ntp_time += kFrameIntervalMs;
2015
2016 // Some resolution constraint should be set.
2017 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2018 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2019 kFrameWidth * kFrameHeight);
2020 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2021
2022 int pixel_count = video_source_.sink_wants().max_pixel_count;
2023 // Triggering a CPU underuse should not change the sink wants since it has
2024 // not been overused for resolution since we changed degradation preference.
2025 video_stream_encoder_->TriggerCpuUnderuse();
2026 video_source_.IncomingCapturedFrame(
2027 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2028 sink_.WaitForEncodedFrame(ntp_time);
2029 ntp_time += kFrameIntervalMs;
2030 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2031 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2032
2033 // Change the degradation preference back. CPU underuse should now adapt.
2034 video_stream_encoder_->SetSource(
2035 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2036 video_source_.IncomingCapturedFrame(
2037 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2038 sink_.WaitForEncodedFrame(ntp_time);
2039 ntp_time += 100;
2040 // Resolution adaptations is gone after changing degradation preference.
2041 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2042 EXPECT_EQ(std::numeric_limits<int>::max(),
2043 video_source_.sink_wants().max_pixel_count);
2044 // The fps adaptation from above is now back.
2045 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2046
2047 // Trigger CPU underuse.
2048 video_stream_encoder_->TriggerCpuUnderuse();
2049 video_source_.IncomingCapturedFrame(
2050 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2051 sink_.WaitForEncodedFrame(ntp_time);
2052 ntp_time += kFrameIntervalMs;
2053 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2054
2055 video_stream_encoder_->Stop();
2056}
2057
mflodmancc3d4422017-08-03 08:27:51 -07002058TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002059 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002060 DataRate::BitsPerSec(kTargetBitrateBps),
2061 DataRate::BitsPerSec(kTargetBitrateBps),
2062 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002063 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002064
sprangc5d62e22017-04-02 23:53:04 -07002065 const int kFrameWidth = 1280;
2066 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002067
Åsa Persson8c1bf952018-09-13 10:42:19 +02002068 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002069
kthelgason5e13d412016-12-01 03:59:51 -08002070 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002071 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002072 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002073 frame_timestamp += kFrameIntervalMs;
2074
perkj803d97f2016-11-01 11:45:46 -07002075 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002076 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002077 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002078 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002079 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002080 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002081
asapersson0944a802017-04-07 00:57:58 -07002082 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002083 // wanted resolution.
2084 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2085 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2086 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002087 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002088
2089 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002090 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002091 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002092 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002093 // Give the encoder queue time to process the change in degradation preference
2094 // by waiting for an encoded frame.
2095 new_video_source.IncomingCapturedFrame(
2096 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2097 sink_.WaitForEncodedFrame(frame_timestamp);
2098 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002099 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02002100 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002101
sprangc5d62e22017-04-02 23:53:04 -07002102 // Force an input frame rate to be available, or the adaptation call won't
2103 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002104 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002105 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002106 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002107 stats_proxy_->SetMockStats(stats);
2108
mflodmancc3d4422017-08-03 08:27:51 -07002109 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002110 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002111 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002112 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002113 frame_timestamp += kFrameIntervalMs;
2114
2115 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002116 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002117 EXPECT_EQ(std::numeric_limits<int>::max(),
2118 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002119 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002120
asapersson02465b82017-04-10 01:12:52 -07002121 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002122 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2123 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002124 // Give the encoder queue time to process the change in degradation preference
2125 // by waiting for an encoded frame.
2126 new_video_source.IncomingCapturedFrame(
2127 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2128 sink_.WaitForEncodedFrame(frame_timestamp);
2129 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002130 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07002131
mflodmancc3d4422017-08-03 08:27:51 -07002132 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002133 new_video_source.IncomingCapturedFrame(
2134 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002135 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002136 frame_timestamp += kFrameIntervalMs;
2137
2138 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02002139 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002140
2141 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002142 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002143 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002144 // Give the encoder queue time to process the change in degradation preference
2145 // by waiting for an encoded frame.
2146 new_video_source.IncomingCapturedFrame(
2147 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2148 sink_.WaitForEncodedFrame(frame_timestamp);
2149 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002150 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2151 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002152 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002153 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002154
2155 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002156 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002157 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002158 // Give the encoder queue time to process the change in degradation preference
2159 // by waiting for an encoded frame.
2160 new_video_source.IncomingCapturedFrame(
2161 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2162 sink_.WaitForEncodedFrame(frame_timestamp);
2163 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002164 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2165 EXPECT_EQ(std::numeric_limits<int>::max(),
2166 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002167 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002168
mflodmancc3d4422017-08-03 08:27:51 -07002169 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002170}
2171
mflodmancc3d4422017-08-03 08:27:51 -07002172TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002174 DataRate::BitsPerSec(kTargetBitrateBps),
2175 DataRate::BitsPerSec(kTargetBitrateBps),
2176 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002177
asaperssonfab67072017-04-04 05:51:49 -07002178 const int kWidth = 1280;
2179 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002180 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002181 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002182 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2183 EXPECT_FALSE(stats.bw_limited_resolution);
2184 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2185
2186 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002187 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002188 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002189 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002190
2191 stats = stats_proxy_->GetStats();
2192 EXPECT_TRUE(stats.bw_limited_resolution);
2193 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2194
2195 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002196 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002197 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002198 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002199
2200 stats = stats_proxy_->GetStats();
2201 EXPECT_FALSE(stats.bw_limited_resolution);
2202 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2203 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2204
mflodmancc3d4422017-08-03 08:27:51 -07002205 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002206}
2207
mflodmancc3d4422017-08-03 08:27:51 -07002208TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002210 DataRate::BitsPerSec(kTargetBitrateBps),
2211 DataRate::BitsPerSec(kTargetBitrateBps),
2212 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002213
2214 const int kWidth = 1280;
2215 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002216 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002217 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002218 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2219 EXPECT_FALSE(stats.cpu_limited_resolution);
2220 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2221
2222 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002223 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002224 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002225 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002226
2227 stats = stats_proxy_->GetStats();
2228 EXPECT_TRUE(stats.cpu_limited_resolution);
2229 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2230
2231 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002232 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002233 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002234 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002235
2236 stats = stats_proxy_->GetStats();
2237 EXPECT_FALSE(stats.cpu_limited_resolution);
2238 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002239 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002240
mflodmancc3d4422017-08-03 08:27:51 -07002241 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002242}
2243
mflodmancc3d4422017-08-03 08:27:51 -07002244TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002245 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002246 DataRate::BitsPerSec(kTargetBitrateBps),
2247 DataRate::BitsPerSec(kTargetBitrateBps),
2248 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002249
asaperssonfab67072017-04-04 05:51:49 -07002250 const int kWidth = 1280;
2251 const int kHeight = 720;
2252 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002253 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002254 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002255 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002256 EXPECT_FALSE(stats.cpu_limited_resolution);
2257 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2258
asaperssonfab67072017-04-04 05:51:49 -07002259 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002260 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002261 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002262 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002263 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002264 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002265 EXPECT_TRUE(stats.cpu_limited_resolution);
2266 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2267
2268 // Set new source with adaptation still enabled.
2269 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002270 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002271 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002272
asaperssonfab67072017-04-04 05:51:49 -07002273 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002274 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002275 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002276 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002277 EXPECT_TRUE(stats.cpu_limited_resolution);
2278 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2279
2280 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002281 video_stream_encoder_->SetSource(&new_video_source,
2282 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002283
asaperssonfab67072017-04-04 05:51:49 -07002284 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002285 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002286 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002287 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002288 EXPECT_FALSE(stats.cpu_limited_resolution);
2289 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2290
2291 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002292 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002293 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002294
asaperssonfab67072017-04-04 05:51:49 -07002295 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002296 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002297 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002298 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002299 EXPECT_TRUE(stats.cpu_limited_resolution);
2300 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2301
asaperssonfab67072017-04-04 05:51:49 -07002302 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002303 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002304 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002305 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002306 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002307 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002308 EXPECT_FALSE(stats.cpu_limited_resolution);
2309 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002310 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002311
mflodmancc3d4422017-08-03 08:27:51 -07002312 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002313}
2314
mflodmancc3d4422017-08-03 08:27:51 -07002315TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002316 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002317 DataRate::BitsPerSec(kTargetBitrateBps),
2318 DataRate::BitsPerSec(kTargetBitrateBps),
2319 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002320
asaperssonfab67072017-04-04 05:51:49 -07002321 const int kWidth = 1280;
2322 const int kHeight = 720;
2323 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002324 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002325 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002326 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002327 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002328 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002329
2330 // Set new source with adaptation still enabled.
2331 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002332 video_stream_encoder_->SetSource(&new_video_source,
2333 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002334
asaperssonfab67072017-04-04 05:51:49 -07002335 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002336 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002337 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002338 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002339 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002340 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002341
asaperssonfab67072017-04-04 05:51:49 -07002342 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002343 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002344 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002345 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002346 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002347 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002348 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002349 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002350
asaperssonfab67072017-04-04 05:51:49 -07002351 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002352 video_stream_encoder_->SetSource(&new_video_source,
2353 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002354
asaperssonfab67072017-04-04 05:51:49 -07002355 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002356 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002357 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002358 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002359 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002360 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002361
asapersson02465b82017-04-10 01:12:52 -07002362 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002363 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002364 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002365
asaperssonfab67072017-04-04 05:51:49 -07002366 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002367 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002368 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002369 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002370 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002371 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2372 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002373
mflodmancc3d4422017-08-03 08:27:51 -07002374 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002375}
2376
mflodmancc3d4422017-08-03 08:27:51 -07002377TEST_F(VideoStreamEncoderTest,
2378 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002379 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002380 DataRate::BitsPerSec(kTargetBitrateBps),
2381 DataRate::BitsPerSec(kTargetBitrateBps),
2382 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002383
2384 const int kWidth = 1280;
2385 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002386 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002387 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002388 video_source_.IncomingCapturedFrame(
2389 CreateFrame(timestamp_ms, kWidth, kHeight));
2390 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002391 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2392 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2393 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2394
2395 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002396 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002397 timestamp_ms += kFrameIntervalMs;
2398 video_source_.IncomingCapturedFrame(
2399 CreateFrame(timestamp_ms, kWidth, kHeight));
2400 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002401 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2402 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2403 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2404
2405 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002406 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002407 timestamp_ms += kFrameIntervalMs;
2408 video_source_.IncomingCapturedFrame(
2409 CreateFrame(timestamp_ms, kWidth, kHeight));
2410 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002411 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2412 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2413 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2414
Niels Möller4db138e2018-04-19 09:04:13 +02002415 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002416 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002417
2418 VideoEncoderConfig video_encoder_config;
2419 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2420 // Make format different, to force recreation of encoder.
2421 video_encoder_config.video_format.parameters["foo"] = "foo";
2422 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002423 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002424 timestamp_ms += kFrameIntervalMs;
2425 video_source_.IncomingCapturedFrame(
2426 CreateFrame(timestamp_ms, kWidth, kHeight));
2427 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002428 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2430 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2431
mflodmancc3d4422017-08-03 08:27:51 -07002432 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002433}
2434
mflodmancc3d4422017-08-03 08:27:51 -07002435TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002436 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002438 DataRate::BitsPerSec(kTargetBitrateBps),
2439 DataRate::BitsPerSec(kTargetBitrateBps),
2440 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2441
2442 const int kWidth = 1280;
2443 const int kHeight = 720;
2444 int sequence = 1;
2445
2446 // Enable BALANCED preference, no initial limitation.
2447 test::FrameForwarder source;
2448 video_stream_encoder_->SetSource(&source,
2449 webrtc::DegradationPreference::BALANCED);
2450 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2451 WaitForEncodedFrame(sequence++);
2452 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2453 EXPECT_FALSE(stats.cpu_limited_resolution);
2454 EXPECT_FALSE(stats.cpu_limited_framerate);
2455 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2456
2457 // Trigger CPU overuse, should now adapt down.
2458 video_stream_encoder_->TriggerCpuOveruse();
2459 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2460 WaitForEncodedFrame(sequence++);
2461 stats = stats_proxy_->GetStats();
2462 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2463
2464 // Set new degradation preference should clear restrictions since we changed
2465 // from BALANCED.
2466 video_stream_encoder_->SetSource(
2467 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2468 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2469 WaitForEncodedFrame(sequence++);
2470 stats = stats_proxy_->GetStats();
2471 EXPECT_FALSE(stats.cpu_limited_resolution);
2472 EXPECT_FALSE(stats.cpu_limited_framerate);
2473 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2474
2475 // Force an input frame rate to be available, or the adaptation call won't
2476 // know what framerate to adapt from.
2477 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2478 mock_stats.input_frame_rate = 30;
2479 stats_proxy_->SetMockStats(mock_stats);
2480 video_stream_encoder_->TriggerCpuOveruse();
2481 stats_proxy_->ResetMockStats();
2482 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2483 WaitForEncodedFrame(sequence++);
2484
2485 // We have now adapted once.
2486 stats = stats_proxy_->GetStats();
2487 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2488
2489 // Back to BALANCED, should clear the restrictions again.
2490 video_stream_encoder_->SetSource(&source,
2491 webrtc::DegradationPreference::BALANCED);
2492 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2493 WaitForEncodedFrame(sequence++);
2494 stats = stats_proxy_->GetStats();
2495 EXPECT_FALSE(stats.cpu_limited_resolution);
2496 EXPECT_FALSE(stats.cpu_limited_framerate);
2497 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2498
2499 video_stream_encoder_->Stop();
2500}
2501
2502TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002503 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002504 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002505 DataRate::BitsPerSec(kTargetBitrateBps),
2506 DataRate::BitsPerSec(kTargetBitrateBps),
2507 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002508
asapersson0944a802017-04-07 00:57:58 -07002509 const int kWidth = 1280;
2510 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002511 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002512
asaperssonfab67072017-04-04 05:51:49 -07002513 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002514 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002515 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002516 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002517 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002518 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2519
asapersson02465b82017-04-10 01:12:52 -07002520 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002521 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002522 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002523 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002524 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002525 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002526 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002527 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2528
2529 // Set new source with adaptation still enabled.
2530 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002531 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002532 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002533
2534 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002535 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002536 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002537 stats = stats_proxy_->GetStats();
2538 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002539 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002540 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2541
sprangc5d62e22017-04-02 23:53:04 -07002542 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002543 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002544 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002545 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002546 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002547 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002548 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002549 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002550 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002551 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002552 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2553
sprangc5d62e22017-04-02 23:53:04 -07002554 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002555 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002556 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2557 mock_stats.input_frame_rate = 30;
2558 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002559 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002560 stats_proxy_->ResetMockStats();
2561
2562 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002563 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002564 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002565
2566 // Framerate now adapted.
2567 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002568 EXPECT_FALSE(stats.cpu_limited_resolution);
2569 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002570 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2571
2572 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002573 video_stream_encoder_->SetSource(&new_video_source,
2574 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002575 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002576 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002577 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002578
2579 stats = stats_proxy_->GetStats();
2580 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002581 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002582 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2583
2584 // Try to trigger overuse. Should not succeed.
2585 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002586 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002587 stats_proxy_->ResetMockStats();
2588
2589 stats = stats_proxy_->GetStats();
2590 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002591 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002592 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2593
2594 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002595 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002596 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002597 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002598 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002599 stats = stats_proxy_->GetStats();
2600 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002601 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002602 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002603
2604 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002605 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002606 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002607 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002608 stats = stats_proxy_->GetStats();
2609 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002610 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002611 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2612
2613 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002614 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002615 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002616 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002617 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002618 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002619 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002620 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002621 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002622 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002623 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2624
2625 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002626 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002627 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002628 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002629 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002630 stats = stats_proxy_->GetStats();
2631 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002632 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002633 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002634 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002635
mflodmancc3d4422017-08-03 08:27:51 -07002636 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002637}
2638
mflodmancc3d4422017-08-03 08:27:51 -07002639TEST_F(VideoStreamEncoderTest,
2640 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002641 const int kWidth = 1280;
2642 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002643 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002644 DataRate::BitsPerSec(kTargetBitrateBps),
2645 DataRate::BitsPerSec(kTargetBitrateBps),
2646 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002647
asaperssonfab67072017-04-04 05:51:49 -07002648 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002649 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002650
asaperssonfab67072017-04-04 05:51:49 -07002651 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002652 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002653
asaperssonfab67072017-04-04 05:51:49 -07002654 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002655 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002656
asaperssonfab67072017-04-04 05:51:49 -07002657 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002658 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002659
kthelgason876222f2016-11-29 01:44:11 -08002660 // Expect a scale down.
2661 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002662 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002663
asapersson02465b82017-04-10 01:12:52 -07002664 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002665 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002666 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002667 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002668
asaperssonfab67072017-04-04 05:51:49 -07002669 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002670 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002671 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002672 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002673
asaperssonfab67072017-04-04 05:51:49 -07002674 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002675 EXPECT_EQ(std::numeric_limits<int>::max(),
2676 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002677
asaperssonfab67072017-04-04 05:51:49 -07002678 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002679 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002680 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002681 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002682
asapersson02465b82017-04-10 01:12:52 -07002683 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002684 EXPECT_EQ(std::numeric_limits<int>::max(),
2685 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002686
mflodmancc3d4422017-08-03 08:27:51 -07002687 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002688}
2689
mflodmancc3d4422017-08-03 08:27:51 -07002690TEST_F(VideoStreamEncoderTest,
2691 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002692 const int kWidth = 1280;
2693 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002695 DataRate::BitsPerSec(kTargetBitrateBps),
2696 DataRate::BitsPerSec(kTargetBitrateBps),
2697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002698
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002699 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002700 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002701 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002702 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002703
2704 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002705 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002706 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002707 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2708 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2709
2710 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002711 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002712 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002713 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2714 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2715 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2716
2717 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002718 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002719 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2720 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2721 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2722
mflodmancc3d4422017-08-03 08:27:51 -07002723 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002724}
2725
mflodmancc3d4422017-08-03 08:27:51 -07002726TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002727 const int kWidth = 1280;
2728 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002729 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002730 DataRate::BitsPerSec(kTargetBitrateBps),
2731 DataRate::BitsPerSec(kTargetBitrateBps),
2732 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002733
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002734 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002735 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002736 video_stream_encoder_->SetSource(&source,
2737 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002738 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2739 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002740 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002741
2742 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002743 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002744 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2745 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2746 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2747 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2748
2749 // Trigger adapt down for same input resolution, expect no change.
2750 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2751 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002752 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002753 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2754 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2756
2757 // Trigger adapt down for larger input resolution, expect no change.
2758 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2759 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002760 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002761 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2762 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2763 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2764
mflodmancc3d4422017-08-03 08:27:51 -07002765 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002766}
2767
mflodmancc3d4422017-08-03 08:27:51 -07002768TEST_F(VideoStreamEncoderTest,
2769 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002770 const int kWidth = 1280;
2771 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002773 DataRate::BitsPerSec(kTargetBitrateBps),
2774 DataRate::BitsPerSec(kTargetBitrateBps),
2775 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002776
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002777 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002778 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002779 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002780 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002781
2782 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002783 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002784 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2787
2788 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002789 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002790 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2793
mflodmancc3d4422017-08-03 08:27:51 -07002794 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002795}
2796
mflodmancc3d4422017-08-03 08:27:51 -07002797TEST_F(VideoStreamEncoderTest,
2798 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002799 const int kWidth = 1280;
2800 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002802 DataRate::BitsPerSec(kTargetBitrateBps),
2803 DataRate::BitsPerSec(kTargetBitrateBps),
2804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002805
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002806 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002807 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002808 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002809 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002810
2811 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002812 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002813 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002814 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002815 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2816
2817 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002818 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002819 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002820 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002821 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2822
mflodmancc3d4422017-08-03 08:27:51 -07002823 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002824}
2825
mflodmancc3d4422017-08-03 08:27:51 -07002826TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002827 const int kWidth = 1280;
2828 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002830 DataRate::BitsPerSec(kTargetBitrateBps),
2831 DataRate::BitsPerSec(kTargetBitrateBps),
2832 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002833
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002834 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002835 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002836 video_stream_encoder_->SetSource(&source,
2837 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002838
2839 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2840 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002841 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2845
2846 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002848 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2850 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2851 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2852
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002854}
2855
mflodmancc3d4422017-08-03 08:27:51 -07002856TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002857 const int kWidth = 1280;
2858 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002860 DataRate::BitsPerSec(kTargetBitrateBps),
2861 DataRate::BitsPerSec(kTargetBitrateBps),
2862 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002863
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002864 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002865 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002866 video_stream_encoder_->SetSource(&source,
2867 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002868
2869 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2870 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002871 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002872 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2873 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2874 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2875
2876 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002878 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002879 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2880 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2881 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2882
mflodmancc3d4422017-08-03 08:27:51 -07002883 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002884}
2885
mflodmancc3d4422017-08-03 08:27:51 -07002886TEST_F(VideoStreamEncoderTest,
2887 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002888 const int kWidth = 1280;
2889 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002891 DataRate::BitsPerSec(kTargetBitrateBps),
2892 DataRate::BitsPerSec(kTargetBitrateBps),
2893 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002894
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002895 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002896 AdaptingFrameForwarder source;
2897 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002898 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002899 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002900
2901 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002902 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002903 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002904 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2905 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2906
2907 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002908 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002909 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002910 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002911 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002912 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2913 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2914
2915 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002916 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002917 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002918 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2919 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2920 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2921
mflodmancc3d4422017-08-03 08:27:51 -07002922 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002923}
2924
mflodmancc3d4422017-08-03 08:27:51 -07002925TEST_F(VideoStreamEncoderTest,
2926 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002927 const int kWidth = 1280;
2928 const int kHeight = 720;
2929 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002931 DataRate::BitsPerSec(kTargetBitrateBps),
2932 DataRate::BitsPerSec(kTargetBitrateBps),
2933 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002934
2935 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2936 stats.input_frame_rate = kInputFps;
2937 stats_proxy_->SetMockStats(stats);
2938
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002939 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002940 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2941 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002942 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002943
2944 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002945 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002946 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2947 sink_.WaitForEncodedFrame(2);
2948 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2949
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002950 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002951 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002952 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002953 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002954 // Give the encoder queue time to process the change in degradation preference
2955 // by waiting for an encoded frame.
2956 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2957 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002958 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002959
2960 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002961 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002962 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2963 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002964 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2965
2966 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002967 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002968 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002969
mflodmancc3d4422017-08-03 08:27:51 -07002970 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002971}
2972
mflodmancc3d4422017-08-03 08:27:51 -07002973TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002974 const int kWidth = 1280;
2975 const int kHeight = 720;
2976 const size_t kNumFrames = 10;
2977
Henrik Boström381d1092020-05-12 18:49:07 +02002978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002979 DataRate::BitsPerSec(kTargetBitrateBps),
2980 DataRate::BitsPerSec(kTargetBitrateBps),
2981 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002982
asaperssond0de2952017-04-21 01:47:31 -07002983 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002984 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002985 video_source_.set_adaptation_enabled(true);
2986
2987 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2988 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2989
2990 int downscales = 0;
2991 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002992 video_source_.IncomingCapturedFrame(
2993 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2994 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002995
asaperssonfab67072017-04-04 05:51:49 -07002996 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002997 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002998 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002999 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003000
3001 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3002 ++downscales;
3003
3004 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3005 EXPECT_EQ(downscales,
3006 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3007 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003008 }
mflodmancc3d4422017-08-03 08:27:51 -07003009 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003010}
3011
mflodmancc3d4422017-08-03 08:27:51 -07003012TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003013 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3014 const int kWidth = 1280;
3015 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003016 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003017 DataRate::BitsPerSec(kTargetBitrateBps),
3018 DataRate::BitsPerSec(kTargetBitrateBps),
3019 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003020
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003021 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003022 AdaptingFrameForwarder source;
3023 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003024 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003025 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003026
Åsa Persson8c1bf952018-09-13 10:42:19 +02003027 int64_t timestamp_ms = kFrameIntervalMs;
3028 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003029 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003030 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003031 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3032 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3033
3034 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003035 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003036 timestamp_ms += kFrameIntervalMs;
3037 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3038 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003039 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003040 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3041 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3042
3043 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003044 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003045 timestamp_ms += kFrameIntervalMs;
3046 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003047 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003048 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003049 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3050 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3051
3052 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003053 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003054 timestamp_ms += kFrameIntervalMs;
3055 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3056 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003057 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003058 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3059 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3060
3061 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003062 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003063 timestamp_ms += kFrameIntervalMs;
3064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003065 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003066 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003067 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3068 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3069
mflodmancc3d4422017-08-03 08:27:51 -07003070 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003071}
3072
mflodmancc3d4422017-08-03 08:27:51 -07003073TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003074 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3075 const int kWidth = 1280;
3076 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003078 DataRate::BitsPerSec(kTargetBitrateBps),
3079 DataRate::BitsPerSec(kTargetBitrateBps),
3080 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003081
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003082 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003083 AdaptingFrameForwarder source;
3084 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003085 video_stream_encoder_->SetSource(&source,
3086 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003087
Åsa Persson8c1bf952018-09-13 10:42:19 +02003088 int64_t timestamp_ms = kFrameIntervalMs;
3089 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003090 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003091 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003092 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3093 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3094
3095 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003096 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003097 timestamp_ms += kFrameIntervalMs;
3098 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3099 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003100 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3101 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3102 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3103
3104 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003105 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003106 timestamp_ms += kFrameIntervalMs;
3107 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003108 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003109 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003110 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3111 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3112
3113 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003114 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003115 timestamp_ms += kFrameIntervalMs;
3116 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3117 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003118 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3119 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3120 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3121
3122 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003123 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003124 timestamp_ms += kFrameIntervalMs;
3125 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003126 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003127 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003128 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3129 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3130
mflodmancc3d4422017-08-03 08:27:51 -07003131 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003132}
3133
Sergey Silkin41c650b2019-10-14 13:12:19 +02003134TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3135 fake_encoder_.SetResolutionBitrateLimits(
3136 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3137
Henrik Boström381d1092020-05-12 18:49:07 +02003138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003139 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3140 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3141 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3142 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003143
3144 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3145 AdaptingFrameForwarder source;
3146 source.set_adaptation_enabled(true);
3147 video_stream_encoder_->SetSource(
3148 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3149
3150 // Insert 720p frame.
3151 int64_t timestamp_ms = kFrameIntervalMs;
3152 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3153 WaitForEncodedFrame(1280, 720);
3154
3155 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003156 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003157 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3158 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3159 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3160 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003161 video_stream_encoder_->TriggerQualityLow();
3162
3163 // Insert 720p frame. It should be downscaled and encoded.
3164 timestamp_ms += kFrameIntervalMs;
3165 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3166 WaitForEncodedFrame(960, 540);
3167
3168 // Trigger adapt up. Higher resolution should not be requested duo to lack
3169 // of bitrate.
3170 video_stream_encoder_->TriggerQualityHigh();
3171 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3172
3173 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003175 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3176 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3177 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3178 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003179
3180 // Trigger adapt up. Higher resolution should be requested.
3181 video_stream_encoder_->TriggerQualityHigh();
3182 VerifyFpsMaxResolutionMax(source.sink_wants());
3183
3184 video_stream_encoder_->Stop();
3185}
3186
3187TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3188 fake_encoder_.SetResolutionBitrateLimits(
3189 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3190
3191 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003192 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003193 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3194 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3195 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3196 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003197
3198 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3199 AdaptingFrameForwarder source;
3200 source.set_adaptation_enabled(true);
3201 video_stream_encoder_->SetSource(
3202 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3203
3204 // Insert 720p frame. It should be dropped and lower resolution should be
3205 // requested.
3206 int64_t timestamp_ms = kFrameIntervalMs;
3207 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3208 ExpectDroppedFrame();
3209 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3210
3211 // Insert 720p frame. It should be downscaled and encoded.
3212 timestamp_ms += kFrameIntervalMs;
3213 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3214 WaitForEncodedFrame(960, 540);
3215
3216 video_stream_encoder_->Stop();
3217}
3218
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003219class BalancedDegradationTest : public VideoStreamEncoderTest {
3220 protected:
3221 void SetupTest() {
3222 // Reset encoder for field trials to take effect.
3223 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003224 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003225
3226 // Enable BALANCED preference.
3227 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003228 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3229 }
3230
3231 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003232 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003233 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3234 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003235 }
3236
Åsa Persson45b176f2019-09-30 11:19:05 +02003237 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003238 timestamp_ms_ += kFrameIntervalMs;
3239 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003240 }
3241
3242 void InsertFrameAndWaitForEncoded() {
3243 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003244 sink_.WaitForEncodedFrame(timestamp_ms_);
3245 }
3246
3247 const int kWidth = 640; // pixels:640x360=230400
3248 const int kHeight = 360;
3249 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3250 int64_t timestamp_ms_ = 0;
3251 AdaptingFrameForwarder source_;
3252};
3253
3254TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3255 test::ScopedFieldTrials field_trials(
3256 "WebRTC-Video-BalancedDegradationSettings/"
3257 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3258 SetupTest();
3259
3260 // Force input frame rate.
3261 const int kInputFps = 24;
3262 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3263 stats.input_frame_rate = kInputFps;
3264 stats_proxy_->SetMockStats(stats);
3265
Åsa Persson45b176f2019-09-30 11:19:05 +02003266 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003267 VerifyFpsMaxResolutionMax(source_.sink_wants());
3268
3269 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003270 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3271 // QP samples.
3272 EXPECT_FALSE(
3273 video_stream_encoder_
3274 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003275 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3276
3277 video_stream_encoder_->Stop();
3278}
3279
3280TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3281 test::ScopedFieldTrials field_trials(
3282 "WebRTC-Video-BalancedDegradationSettings/"
3283 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3284 SetupTest();
3285
3286 // Force input frame rate.
3287 const int kInputFps = 25;
3288 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3289 stats.input_frame_rate = kInputFps;
3290 stats_proxy_->SetMockStats(stats);
3291
Åsa Persson45b176f2019-09-30 11:19:05 +02003292 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003293 VerifyFpsMaxResolutionMax(source_.sink_wants());
3294
3295 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003296 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3297 // samples.
3298 EXPECT_TRUE(
3299 video_stream_encoder_
3300 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003301 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3302
3303 video_stream_encoder_->Stop();
3304}
3305
3306TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3307 test::ScopedFieldTrials field_trials(
3308 "WebRTC-Video-BalancedDegradationSettings/"
3309 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3310 SetupTest();
3311
3312 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3313
Åsa Persson45b176f2019-09-30 11:19:05 +02003314 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003315 VerifyFpsMaxResolutionMax(source_.sink_wants());
3316
3317 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3318 video_stream_encoder_->TriggerQualityLow();
3319 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3320
3321 video_stream_encoder_->Stop();
3322}
3323
Åsa Perssonccfb3402019-09-25 15:13:04 +02003324TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003325 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003326 "WebRTC-Video-BalancedDegradationSettings/"
3327 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003328 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003329
Åsa Persson1b247f12019-08-14 17:26:39 +02003330 const int kMinBitrateBps = 425000;
3331 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003332 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003333
Åsa Persson45b176f2019-09-30 11:19:05 +02003334 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003335 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003336 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3337
3338 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3339 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003340 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003341 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003342 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3343
3344 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3345 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003346 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003347 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003348 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3349
Åsa Persson30ab0152019-08-27 12:22:33 +02003350 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3351 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003352 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003353 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3354 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003355 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3356
3357 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003358 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003359 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003360 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003361
Åsa Persson30ab0152019-08-27 12:22:33 +02003362 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003363 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003364 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003365 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003366 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003367 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3368
3369 video_stream_encoder_->Stop();
3370}
3371
Åsa Perssonccfb3402019-09-25 15:13:04 +02003372TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003373 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3374 test::ScopedFieldTrials field_trials(
3375 "WebRTC-Video-BalancedDegradationSettings/"
3376 "pixels:57600|129600|230400,fps:7|24|24/");
3377 SetupTest();
3378 OnBitrateUpdated(kLowTargetBitrateBps);
3379
3380 VerifyNoLimitation(source_.sink_wants());
3381
3382 // Insert frame, expect scaled down:
3383 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3384 InsertFrame();
3385 EXPECT_FALSE(WaitForFrame(1000));
3386 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3387 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3388
3389 // Insert frame, expect scaled down:
3390 // resolution (320x180@24fps).
3391 InsertFrame();
3392 EXPECT_FALSE(WaitForFrame(1000));
3393 EXPECT_LT(source_.sink_wants().max_pixel_count,
3394 source_.last_wants().max_pixel_count);
3395 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3396
3397 // Frame should not be dropped (min pixels per frame reached).
3398 InsertFrameAndWaitForEncoded();
3399
3400 video_stream_encoder_->Stop();
3401}
3402
3403TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003404 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003405 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003406 "WebRTC-Video-BalancedDegradationSettings/"
3407 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003408 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003409
Åsa Persson30ab0152019-08-27 12:22:33 +02003410 const int kResolutionMinBitrateBps = 435000;
3411 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003412 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003413
Åsa Persson45b176f2019-09-30 11:19:05 +02003414 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003415 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003416 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3417
3418 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3419 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003420 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003421 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003422 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3425 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003426 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003427 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003428 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
3430 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3431 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003432 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003433 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003434 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
Åsa Persson30ab0152019-08-27 12:22:33 +02003436 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3437 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003438 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003439 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003440 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3441
3442 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3443 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003444 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003445 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3446
3447 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003448 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003449 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003450 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003451 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003452 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3453
3454 video_stream_encoder_->Stop();
3455}
3456
Åsa Perssonccfb3402019-09-25 15:13:04 +02003457TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003458 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003459 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003460 "WebRTC-Video-BalancedDegradationSettings/"
3461 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003462 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003463
Åsa Persson30ab0152019-08-27 12:22:33 +02003464 const int kMinBitrateBps = 425000;
3465 const int kTooLowMinBitrateBps = 424000;
3466 const int kResolutionMinBitrateBps = 435000;
3467 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003468 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003469
Åsa Persson45b176f2019-09-30 11:19:05 +02003470 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003471 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003472 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3473
3474 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3475 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003476 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003477 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003478 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3479
3480 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3481 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003482 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003483 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003484 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3485
3486 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3487 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003488 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003489 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003490 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3491
3492 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3493 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003494 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003495 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3496
3497 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003498 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003499 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003500 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003501 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003502 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3503
3504 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003505 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003506 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003507 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003508 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3509
3510 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003511 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003512 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003513 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003514 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003515 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3516
Åsa Persson1b247f12019-08-14 17:26:39 +02003517 video_stream_encoder_->Stop();
3518}
3519
mflodmancc3d4422017-08-03 08:27:51 -07003520TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003521 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3522 const int kWidth = 1280;
3523 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003524 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003525 DataRate::BitsPerSec(kTargetBitrateBps),
3526 DataRate::BitsPerSec(kTargetBitrateBps),
3527 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003528
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003529 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003530 AdaptingFrameForwarder source;
3531 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003532 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003533 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003534
Åsa Persson8c1bf952018-09-13 10:42:19 +02003535 int64_t timestamp_ms = kFrameIntervalMs;
3536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003537 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003538 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003539 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3541 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3542 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3543
3544 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003545 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003546 timestamp_ms += kFrameIntervalMs;
3547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3548 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003549 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003550 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3552 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3553 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3554
3555 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003556 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003557 timestamp_ms += kFrameIntervalMs;
3558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3559 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003560 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003561 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3562 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3563 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3564 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3565
Jonathan Yubc771b72017-12-08 17:04:29 -08003566 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003567 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003568 timestamp_ms += kFrameIntervalMs;
3569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3570 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003571 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003572 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003574 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003575 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3576
Jonathan Yubc771b72017-12-08 17:04:29 -08003577 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003578 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003579 timestamp_ms += kFrameIntervalMs;
3580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3581 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003582 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003583 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003584 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3587 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3588
Jonathan Yubc771b72017-12-08 17:04:29 -08003589 // Trigger quality adapt down, expect no change (min resolution reached).
3590 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3593 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003594 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3595 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3597 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3598 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3599
3600 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003601 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003602 timestamp_ms += kFrameIntervalMs;
3603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3604 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003605 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003606 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3608 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3609 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3610
3611 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003612 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003613 timestamp_ms += kFrameIntervalMs;
3614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3615 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003616 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3617 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3619 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3620 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3621
3622 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003623 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003624 timestamp_ms += kFrameIntervalMs;
3625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3626 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003627 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003628 last_wants = source.sink_wants();
3629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003631 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003632 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3633
3634 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003635 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003636 timestamp_ms += kFrameIntervalMs;
3637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3638 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003639 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003642 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003643 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3644
3645 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003646 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003647 timestamp_ms += kFrameIntervalMs;
3648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003649 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003650 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003651 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003654 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003655 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003656
mflodmancc3d4422017-08-03 08:27:51 -07003657 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003658}
3659
mflodmancc3d4422017-08-03 08:27:51 -07003660TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003661 const int kWidth = 640;
3662 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003663
Henrik Boström381d1092020-05-12 18:49:07 +02003664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003665 DataRate::BitsPerSec(kTargetBitrateBps),
3666 DataRate::BitsPerSec(kTargetBitrateBps),
3667 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003668
perkj803d97f2016-11-01 11:45:46 -07003669 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003670 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003671 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003672 }
3673
mflodmancc3d4422017-08-03 08:27:51 -07003674 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003675 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003676 video_source_.IncomingCapturedFrame(CreateFrame(
3677 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003678 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003679 }
3680
mflodmancc3d4422017-08-03 08:27:51 -07003681 video_stream_encoder_->Stop();
3682 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003683 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003684
Ying Wangef3998f2019-12-09 13:06:53 +01003685 EXPECT_METRIC_EQ(
3686 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3687 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003688 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3689}
3690
mflodmancc3d4422017-08-03 08:27:51 -07003691TEST_F(VideoStreamEncoderTest,
3692 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003693 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003694 DataRate::BitsPerSec(kTargetBitrateBps),
3695 DataRate::BitsPerSec(kTargetBitrateBps),
3696 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003697 const int kWidth = 640;
3698 const int kHeight = 360;
3699
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003700 video_stream_encoder_->SetSource(&video_source_,
3701 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003702
3703 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3704 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003705 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003706 }
3707
mflodmancc3d4422017-08-03 08:27:51 -07003708 video_stream_encoder_->Stop();
3709 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003710 stats_proxy_.reset();
3711
3712 EXPECT_EQ(0,
3713 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3714}
3715
mflodmancc3d4422017-08-03 08:27:51 -07003716TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003717 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003718 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003719
3720 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003721 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003722 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003723 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3724 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003725
sprang57c2fff2017-01-16 06:24:02 -08003726 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003727 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003729 DataRate::BitsPerSec(kLowTargetBitrateBps),
3730 DataRate::BitsPerSec(kLowTargetBitrateBps),
3731 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003732
sprang57c2fff2017-01-16 06:24:02 -08003733 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003734 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3735 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003736 VideoBitrateAllocation bitrate_allocation =
3737 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003738 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003739 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003740 // TODO(srte): The use of millisecs here looks like an error, but the tests
3741 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003742 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003743
3744 // Not called on second frame.
3745 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3746 .Times(0);
3747 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003748 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3749 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003750 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003751
3752 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003753 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3754 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003755 const int64_t start_time_ms = rtc::TimeMillis();
3756 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3757 video_source_.IncomingCapturedFrame(
3758 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3759 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003760 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003761 }
3762
3763 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003764 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003765
mflodmancc3d4422017-08-03 08:27:51 -07003766 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003767}
3768
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003769TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3770 // 2 TLs configured, temporal layers supported by encoder.
3771 const int kNumTemporalLayers = 2;
3772 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3773 fake_encoder_.SetTemporalLayersSupported(0, true);
3774
3775 // Bitrate allocated across temporal layers.
3776 const int kTl0Bps = kTargetBitrateBps *
3777 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003778 kNumTemporalLayers, /*temporal_id*/ 0,
3779 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003780 const int kTl1Bps = kTargetBitrateBps *
3781 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003782 kNumTemporalLayers, /*temporal_id*/ 1,
3783 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003784 VideoBitrateAllocation expected_bitrate;
3785 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3786 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3787
3788 VerifyAllocatedBitrate(expected_bitrate);
3789 video_stream_encoder_->Stop();
3790}
3791
3792TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3793 // 2 TLs configured, temporal layers not supported by encoder.
3794 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3795 fake_encoder_.SetTemporalLayersSupported(0, false);
3796
3797 // Temporal layers not supported by the encoder.
3798 // Total bitrate should be at ti:0.
3799 VideoBitrateAllocation expected_bitrate;
3800 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3801
3802 VerifyAllocatedBitrate(expected_bitrate);
3803 video_stream_encoder_->Stop();
3804}
3805
3806TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3807 // 2 TLs configured, temporal layers only supported for first stream.
3808 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3809 fake_encoder_.SetTemporalLayersSupported(0, true);
3810 fake_encoder_.SetTemporalLayersSupported(1, false);
3811
3812 const int kS0Bps = 150000;
3813 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003814 kS0Bps *
3815 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3816 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003817 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003818 kS0Bps *
3819 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3820 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003821 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3822 // Temporal layers not supported by si:1.
3823 VideoBitrateAllocation expected_bitrate;
3824 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3825 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3826 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3827
3828 VerifyAllocatedBitrate(expected_bitrate);
3829 video_stream_encoder_->Stop();
3830}
3831
Niels Möller7dc26b72017-12-06 10:27:48 +01003832TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3833 const int kFrameWidth = 1280;
3834 const int kFrameHeight = 720;
3835 const int kFramerate = 24;
3836
Henrik Boström381d1092020-05-12 18:49:07 +02003837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003838 DataRate::BitsPerSec(kTargetBitrateBps),
3839 DataRate::BitsPerSec(kTargetBitrateBps),
3840 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003841 test::FrameForwarder source;
3842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003843 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003844
3845 // Insert a single frame, triggering initial configuration.
3846 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3848
3849 EXPECT_EQ(
3850 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3851 kDefaultFramerate);
3852
3853 // Trigger reconfigure encoder (without resetting the entire instance).
3854 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003855 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003856 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3857 video_encoder_config.number_of_streams = 1;
3858 video_encoder_config.video_stream_factory =
3859 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3860 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003861 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3863
3864 // Detector should be updated with fps limit from codec config.
3865 EXPECT_EQ(
3866 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3867 kFramerate);
3868
3869 // Trigger overuse, max framerate should be reduced.
3870 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3871 stats.input_frame_rate = kFramerate;
3872 stats_proxy_->SetMockStats(stats);
3873 video_stream_encoder_->TriggerCpuOveruse();
3874 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3875 int adapted_framerate =
3876 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3877 EXPECT_LT(adapted_framerate, kFramerate);
3878
3879 // Trigger underuse, max framerate should go back to codec configured fps.
3880 // Set extra low fps, to make sure it's actually reset, not just incremented.
3881 stats = stats_proxy_->GetStats();
3882 stats.input_frame_rate = adapted_framerate / 2;
3883 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003884 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003885 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3886 EXPECT_EQ(
3887 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3888 kFramerate);
3889
3890 video_stream_encoder_->Stop();
3891}
3892
3893TEST_F(VideoStreamEncoderTest,
3894 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3895 const int kFrameWidth = 1280;
3896 const int kFrameHeight = 720;
3897 const int kLowFramerate = 15;
3898 const int kHighFramerate = 25;
3899
Henrik Boström381d1092020-05-12 18:49:07 +02003900 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003901 DataRate::BitsPerSec(kTargetBitrateBps),
3902 DataRate::BitsPerSec(kTargetBitrateBps),
3903 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003904 test::FrameForwarder source;
3905 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003906 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003907
3908 // Trigger initial configuration.
3909 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003910 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003911 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3912 video_encoder_config.number_of_streams = 1;
3913 video_encoder_config.video_stream_factory =
3914 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3915 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3916 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003917 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003918 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3919
3920 EXPECT_EQ(
3921 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3922 kLowFramerate);
3923
3924 // Trigger overuse, max framerate should be reduced.
3925 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3926 stats.input_frame_rate = kLowFramerate;
3927 stats_proxy_->SetMockStats(stats);
3928 video_stream_encoder_->TriggerCpuOveruse();
3929 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3930 int adapted_framerate =
3931 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3932 EXPECT_LT(adapted_framerate, kLowFramerate);
3933
3934 // Reconfigure the encoder with a new (higher max framerate), max fps should
3935 // still respect the adaptation.
3936 video_encoder_config.video_stream_factory =
3937 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3938 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3939 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003940 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003941 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3942
3943 EXPECT_EQ(
3944 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3945 adapted_framerate);
3946
3947 // Trigger underuse, max framerate should go back to codec configured fps.
3948 stats = stats_proxy_->GetStats();
3949 stats.input_frame_rate = adapted_framerate;
3950 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003951 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003952 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3953 EXPECT_EQ(
3954 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3955 kHighFramerate);
3956
3957 video_stream_encoder_->Stop();
3958}
3959
mflodmancc3d4422017-08-03 08:27:51 -07003960TEST_F(VideoStreamEncoderTest,
3961 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003962 const int kFrameWidth = 1280;
3963 const int kFrameHeight = 720;
3964 const int kFramerate = 24;
3965
Henrik Boström381d1092020-05-12 18:49:07 +02003966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003967 DataRate::BitsPerSec(kTargetBitrateBps),
3968 DataRate::BitsPerSec(kTargetBitrateBps),
3969 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003970 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003971 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003972 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003973
3974 // Trigger initial configuration.
3975 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003976 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003977 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3978 video_encoder_config.number_of_streams = 1;
3979 video_encoder_config.video_stream_factory =
3980 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3981 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003982 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003983 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003984 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003985
Niels Möller7dc26b72017-12-06 10:27:48 +01003986 EXPECT_EQ(
3987 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3988 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003989
3990 // Trigger overuse, max framerate should be reduced.
3991 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3992 stats.input_frame_rate = kFramerate;
3993 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003994 video_stream_encoder_->TriggerCpuOveruse();
3995 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003996 int adapted_framerate =
3997 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003998 EXPECT_LT(adapted_framerate, kFramerate);
3999
4000 // Change degradation preference to not enable framerate scaling. Target
4001 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004002 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004003 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004004 EXPECT_EQ(
4005 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4006 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004007
mflodmancc3d4422017-08-03 08:27:51 -07004008 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004009}
4010
mflodmancc3d4422017-08-03 08:27:51 -07004011TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004012 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004014 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4015 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4016 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004017 const int kWidth = 640;
4018 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004019
asaperssonfab67072017-04-04 05:51:49 -07004020 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004021
4022 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004023 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004024
4025 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07004026 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004027
sprangc5d62e22017-04-02 23:53:04 -07004028 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004029
asaperssonfab67072017-04-04 05:51:49 -07004030 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004031 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004032 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004033
4034 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004035 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004036
sprangc5d62e22017-04-02 23:53:04 -07004037 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08004038
mflodmancc3d4422017-08-03 08:27:51 -07004039 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004040}
4041
mflodmancc3d4422017-08-03 08:27:51 -07004042TEST_F(VideoStreamEncoderTest,
4043 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004044 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004046 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4047 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4048 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004049 const int kWidth = 640;
4050 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004051
4052 // We expect the n initial frames to get dropped.
4053 int i;
4054 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004055 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004056 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004057 }
4058 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004059 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004060 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004061
4062 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004063 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004064
mflodmancc3d4422017-08-03 08:27:51 -07004065 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004066}
4067
mflodmancc3d4422017-08-03 08:27:51 -07004068TEST_F(VideoStreamEncoderTest,
4069 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004070 const int kWidth = 640;
4071 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004073 DataRate::BitsPerSec(kLowTargetBitrateBps),
4074 DataRate::BitsPerSec(kLowTargetBitrateBps),
4075 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004076
4077 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004078 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004079 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004080
asaperssonfab67072017-04-04 05:51:49 -07004081 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004082 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004083 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004084
mflodmancc3d4422017-08-03 08:27:51 -07004085 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004086}
4087
mflodmancc3d4422017-08-03 08:27:51 -07004088TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004089 const int kWidth = 640;
4090 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004091 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004092
4093 VideoEncoderConfig video_encoder_config;
4094 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4095 // Make format different, to force recreation of encoder.
4096 video_encoder_config.video_format.parameters["foo"] = "foo";
4097 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004098 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004100 DataRate::BitsPerSec(kLowTargetBitrateBps),
4101 DataRate::BitsPerSec(kLowTargetBitrateBps),
4102 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004103
kthelgasonb83797b2017-02-14 11:57:25 -08004104 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004105 video_stream_encoder_->SetSource(&video_source_,
4106 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004107
asaperssonfab67072017-04-04 05:51:49 -07004108 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004109 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004110 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004111
mflodmancc3d4422017-08-03 08:27:51 -07004112 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004113 fake_encoder_.SetQualityScaling(true);
4114}
4115
Åsa Persson139f4dc2019-08-02 09:29:58 +02004116TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4117 webrtc::test::ScopedFieldTrials field_trials(
4118 "WebRTC-Video-QualityScalerSettings/"
4119 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4120 // Reset encoder for field trials to take effect.
4121 ConfigureEncoder(video_encoder_config_.Copy());
4122 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4123 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4124 const int kWidth = 640;
4125 const int kHeight = 360;
4126
Henrik Boström381d1092020-05-12 18:49:07 +02004127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004128 DataRate::BitsPerSec(kTargetBitrateBps),
4129 DataRate::BitsPerSec(kTargetBitrateBps),
4130 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004131 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4132 // Frame should not be dropped.
4133 WaitForEncodedFrame(1);
4134
Henrik Boström381d1092020-05-12 18:49:07 +02004135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004136 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4137 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4138 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004139 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4140 // Frame should not be dropped.
4141 WaitForEncodedFrame(2);
4142
Henrik Boström381d1092020-05-12 18:49:07 +02004143 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004144 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4145 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4146 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004147 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4148 // Expect to drop this frame, the wait should time out.
4149 ExpectDroppedFrame();
4150
4151 // Expect the sink_wants to specify a scaled frame.
4152 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
4153 video_stream_encoder_->Stop();
4154}
4155
Åsa Perssone644a032019-11-08 15:56:00 +01004156TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4157 webrtc::test::ScopedFieldTrials field_trials(
4158 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4159
4160 // Reset encoder for field trials to take effect.
4161 VideoEncoderConfig config = video_encoder_config_.Copy();
4162 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004163 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004164 ConfigureEncoder(std::move(config));
4165 fake_encoder_.SetQp(kQpLow);
4166
4167 // Enable MAINTAIN_FRAMERATE preference.
4168 AdaptingFrameForwarder source;
4169 source.set_adaptation_enabled(true);
4170 video_stream_encoder_->SetSource(&source,
4171 DegradationPreference::MAINTAIN_FRAMERATE);
4172
4173 // Start at low bitrate.
4174 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4176 DataRate::BitsPerSec(kLowBitrateBps),
4177 DataRate::BitsPerSec(kLowBitrateBps),
4178 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004179
4180 // Expect first frame to be dropped and resolution to be limited.
4181 const int kWidth = 1280;
4182 const int kHeight = 720;
4183 const int64_t kFrameIntervalMs = 100;
4184 int64_t timestamp_ms = kFrameIntervalMs;
4185 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4186 ExpectDroppedFrame();
4187 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4188
4189 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4191 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004192
4193 // Insert frames and advance |min_duration_ms|.
4194 for (size_t i = 1; i <= 10; i++) {
4195 timestamp_ms += kFrameIntervalMs;
4196 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4197 WaitForEncodedFrame(timestamp_ms);
4198 }
4199 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4200 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4201
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004202 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004203
4204 // Insert frame should trigger high BW and release quality limitation.
4205 timestamp_ms += kFrameIntervalMs;
4206 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4207 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004208 // The ramp-up code involves the adaptation queue, give it time to execute.
4209 // TODO(hbos): Can we await an appropriate event instead?
4210 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Åsa Perssone644a032019-11-08 15:56:00 +01004211 VerifyFpsMaxResolutionMax(source.sink_wants());
4212
4213 // Frame should not be adapted.
4214 timestamp_ms += kFrameIntervalMs;
4215 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4216 WaitForEncodedFrame(kWidth, kHeight);
4217 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4218
4219 video_stream_encoder_->Stop();
4220}
4221
mflodmancc3d4422017-08-03 08:27:51 -07004222TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004223 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4224 const int kTooSmallWidth = 10;
4225 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004226 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004227 DataRate::BitsPerSec(kTargetBitrateBps),
4228 DataRate::BitsPerSec(kTargetBitrateBps),
4229 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004230
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004231 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004232 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004233 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004234 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004235 VerifyNoLimitation(source.sink_wants());
4236 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4237
4238 // Trigger adapt down, too small frame, expect no change.
4239 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004240 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004241 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004242 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07004243 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4244 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4245
mflodmancc3d4422017-08-03 08:27:51 -07004246 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004247}
4248
mflodmancc3d4422017-08-03 08:27:51 -07004249TEST_F(VideoStreamEncoderTest,
4250 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004251 const int kTooSmallWidth = 10;
4252 const int kTooSmallHeight = 10;
4253 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004255 DataRate::BitsPerSec(kTargetBitrateBps),
4256 DataRate::BitsPerSec(kTargetBitrateBps),
4257 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004258
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004259 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004260 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004261 video_stream_encoder_->SetSource(&source,
4262 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004263 VerifyNoLimitation(source.sink_wants());
4264 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4265 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4266
4267 // Trigger adapt down, expect limited framerate.
4268 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004269 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004270 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004271 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4272 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4273 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4274 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4275
4276 // Trigger adapt down, too small frame, expect no change.
4277 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004278 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004279 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004280 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4281 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4282 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4283 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4284
mflodmancc3d4422017-08-03 08:27:51 -07004285 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004286}
4287
mflodmancc3d4422017-08-03 08:27:51 -07004288TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004289 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004291 DataRate::BitsPerSec(kTargetBitrateBps),
4292 DataRate::BitsPerSec(kTargetBitrateBps),
4293 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004294 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004295 const int kFrameWidth = 1280;
4296 const int kFrameHeight = 720;
4297 video_source_.IncomingCapturedFrame(
4298 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004299 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004300 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004301}
4302
sprangb1ca0732017-02-01 08:38:12 -08004303// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004304TEST_F(VideoStreamEncoderTest,
4305 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004306 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004307 DataRate::BitsPerSec(kTargetBitrateBps),
4308 DataRate::BitsPerSec(kTargetBitrateBps),
4309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004310
4311 const int kFrameWidth = 1280;
4312 const int kFrameHeight = 720;
4313 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004314 // requested by
4315 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004316 video_source_.set_adaptation_enabled(true);
4317
4318 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004319 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004320 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004321
4322 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004323 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004324 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004325 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004326 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004327
asaperssonfab67072017-04-04 05:51:49 -07004328 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004329 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004330 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004331 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004332 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004333
mflodmancc3d4422017-08-03 08:27:51 -07004334 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004335}
sprangfe627f32017-03-29 08:24:59 -07004336
mflodmancc3d4422017-08-03 08:27:51 -07004337TEST_F(VideoStreamEncoderTest,
4338 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004339 const int kFrameWidth = 1280;
4340 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004341
Henrik Boström381d1092020-05-12 18:49:07 +02004342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004343 DataRate::BitsPerSec(kTargetBitrateBps),
4344 DataRate::BitsPerSec(kTargetBitrateBps),
4345 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004346 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004347 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004348 video_source_.set_adaptation_enabled(true);
4349
sprang4847ae62017-06-27 07:06:52 -07004350 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004351
4352 video_source_.IncomingCapturedFrame(
4353 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004354 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004355
4356 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004357 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004358
4359 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004360 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004361 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004362 video_source_.IncomingCapturedFrame(
4363 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004364 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004365 }
4366
4367 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004368 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004369 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004370 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004371 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004372 video_source_.IncomingCapturedFrame(
4373 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004374 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004375 ++num_frames_dropped;
4376 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004377 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004378 }
4379 }
4380
sprang4847ae62017-06-27 07:06:52 -07004381 // Add some slack to account for frames dropped by the frame dropper.
4382 const int kErrorMargin = 1;
4383 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004384 kErrorMargin);
4385
4386 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004387 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004388 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004389 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004390 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004391 video_source_.IncomingCapturedFrame(
4392 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004393 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004394 ++num_frames_dropped;
4395 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004396 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004397 }
4398 }
sprang4847ae62017-06-27 07:06:52 -07004399 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004400 kErrorMargin);
4401
4402 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004403 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004404 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004405 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004406 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004407 video_source_.IncomingCapturedFrame(
4408 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004409 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004410 ++num_frames_dropped;
4411 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004412 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004413 }
4414 }
sprang4847ae62017-06-27 07:06:52 -07004415 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004416 kErrorMargin);
4417
4418 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004419 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004420 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004421 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004422 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004423 video_source_.IncomingCapturedFrame(
4424 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004425 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004426 ++num_frames_dropped;
4427 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004428 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004429 }
4430 }
4431 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4432
mflodmancc3d4422017-08-03 08:27:51 -07004433 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004434}
4435
mflodmancc3d4422017-08-03 08:27:51 -07004436TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004437 const int kFramerateFps = 5;
4438 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004439 const int kFrameWidth = 1280;
4440 const int kFrameHeight = 720;
4441
sprang4847ae62017-06-27 07:06:52 -07004442 // Reconfigure encoder with two temporal layers and screensharing, which will
4443 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004444 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004445
Henrik Boström381d1092020-05-12 18:49:07 +02004446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004447 DataRate::BitsPerSec(kTargetBitrateBps),
4448 DataRate::BitsPerSec(kTargetBitrateBps),
4449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004450 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004451 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004452 video_source_.set_adaptation_enabled(true);
4453
sprang4847ae62017-06-27 07:06:52 -07004454 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004455
4456 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004457 rtc::VideoSinkWants last_wants;
4458 do {
4459 last_wants = video_source_.sink_wants();
4460
sprangc5d62e22017-04-02 23:53:04 -07004461 // Insert frames to get a new fps estimate...
4462 for (int j = 0; j < kFramerateFps; ++j) {
4463 video_source_.IncomingCapturedFrame(
4464 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004465 if (video_source_.last_sent_width()) {
4466 sink_.WaitForEncodedFrame(timestamp_ms);
4467 }
sprangc5d62e22017-04-02 23:53:04 -07004468 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004469 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004470 }
4471 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004472 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004473 } while (video_source_.sink_wants().max_framerate_fps <
4474 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004475
Jonathan Yubc771b72017-12-08 17:04:29 -08004476 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004477
mflodmancc3d4422017-08-03 08:27:51 -07004478 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004479}
asaperssonf7e294d2017-06-13 23:25:22 -07004480
mflodmancc3d4422017-08-03 08:27:51 -07004481TEST_F(VideoStreamEncoderTest,
4482 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004483 const int kWidth = 1280;
4484 const int kHeight = 720;
4485 const int64_t kFrameIntervalMs = 150;
4486 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004488 DataRate::BitsPerSec(kTargetBitrateBps),
4489 DataRate::BitsPerSec(kTargetBitrateBps),
4490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004491
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004492 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004493 AdaptingFrameForwarder source;
4494 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004495 video_stream_encoder_->SetSource(&source,
4496 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004497 timestamp_ms += kFrameIntervalMs;
4498 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004499 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004500 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4503 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4504
4505 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004506 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004507 timestamp_ms += kFrameIntervalMs;
4508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004509 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004510 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4512 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4513 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4514
4515 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004516 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004517 timestamp_ms += kFrameIntervalMs;
4518 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004519 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004520 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4522 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4523 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4524
4525 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004526 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004527 timestamp_ms += kFrameIntervalMs;
4528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004529 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004530 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4532 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4533 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4534
4535 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004536 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004537 timestamp_ms += kFrameIntervalMs;
4538 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004539 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004540 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4543 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4544
4545 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004546 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004547 timestamp_ms += kFrameIntervalMs;
4548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004549 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004550 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4553 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4554
4555 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004556 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004557 timestamp_ms += kFrameIntervalMs;
4558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004559 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004560 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4563 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4564
4565 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004566 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004567 timestamp_ms += kFrameIntervalMs;
4568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004569 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004570 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4571 rtc::VideoSinkWants last_wants = source.sink_wants();
4572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4573 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4574 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4575
4576 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004577 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004578 timestamp_ms += kFrameIntervalMs;
4579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004580 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004581 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4584 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4585
4586 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004587 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004588 timestamp_ms += kFrameIntervalMs;
4589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004590 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004591 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4594 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4595
4596 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004597 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004598 timestamp_ms += kFrameIntervalMs;
4599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004600 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004601 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4604 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4605
4606 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004607 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004608 timestamp_ms += kFrameIntervalMs;
4609 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004610 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004611 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4614 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4615
4616 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004617 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004618 timestamp_ms += kFrameIntervalMs;
4619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004620 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004621 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4624 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4625
4626 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004627 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004628 timestamp_ms += kFrameIntervalMs;
4629 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004630 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004631 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4634 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4635
4636 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004637 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004638 timestamp_ms += kFrameIntervalMs;
4639 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004640 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004641 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4643 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4644 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4645
Åsa Persson30ab0152019-08-27 12:22:33 +02004646 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004647 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004648 timestamp_ms += kFrameIntervalMs;
4649 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004650 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004651 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004652 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4655 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4656
4657 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004658 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004659 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004660 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4661
mflodmancc3d4422017-08-03 08:27:51 -07004662 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004663}
4664
mflodmancc3d4422017-08-03 08:27:51 -07004665TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004666 const int kWidth = 1280;
4667 const int kHeight = 720;
4668 const int64_t kFrameIntervalMs = 150;
4669 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004671 DataRate::BitsPerSec(kTargetBitrateBps),
4672 DataRate::BitsPerSec(kTargetBitrateBps),
4673 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004674
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004675 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004676 AdaptingFrameForwarder source;
4677 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004678 video_stream_encoder_->SetSource(&source,
4679 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004680 timestamp_ms += kFrameIntervalMs;
4681 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004682 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004683 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004684 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4685 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4686 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4687 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4688 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4689 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4690
4691 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004692 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004693 timestamp_ms += kFrameIntervalMs;
4694 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004695 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004696 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4697 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4698 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4699 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4700 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4701 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4703
4704 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004705 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004706 timestamp_ms += kFrameIntervalMs;
4707 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004708 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004709 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4711 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4712 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4713 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4714 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4715 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4716
4717 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004718 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004719 timestamp_ms += kFrameIntervalMs;
4720 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004721 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004722 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4724 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4725 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4726 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4727 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4728 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4729
4730 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004731 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004732 timestamp_ms += kFrameIntervalMs;
4733 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004734 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004735 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4736 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4738 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4739 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4740 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4741 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4742
4743 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004744 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004745 timestamp_ms += kFrameIntervalMs;
4746 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004747 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004748 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4749 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4750 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4751 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4753 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4754 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4755
4756 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004757 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004758 timestamp_ms += kFrameIntervalMs;
4759 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004760 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004761 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004762 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004763 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4764 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4765 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4766 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4767 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4768 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4769
4770 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004771 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004772 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004773 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4774 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4775
mflodmancc3d4422017-08-03 08:27:51 -07004776 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004777}
4778
mflodmancc3d4422017-08-03 08:27:51 -07004779TEST_F(VideoStreamEncoderTest,
4780 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004781 const int kWidth = 640;
4782 const int kHeight = 360;
4783 const int kFpsLimit = 15;
4784 const int64_t kFrameIntervalMs = 150;
4785 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004787 DataRate::BitsPerSec(kTargetBitrateBps),
4788 DataRate::BitsPerSec(kTargetBitrateBps),
4789 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004790
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004791 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004792 AdaptingFrameForwarder source;
4793 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004794 video_stream_encoder_->SetSource(&source,
4795 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004796 timestamp_ms += kFrameIntervalMs;
4797 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004798 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004799 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004800 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4801 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4803 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4804 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4805 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4806
4807 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004808 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004809 timestamp_ms += kFrameIntervalMs;
4810 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004811 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004812 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4813 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4814 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4816 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4817 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4818 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4819
4820 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004821 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004822 timestamp_ms += kFrameIntervalMs;
4823 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004824 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004825 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4826 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4828 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4829 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4830 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4831 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4832
4833 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004834 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004835 timestamp_ms += kFrameIntervalMs;
4836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004837 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004838 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4839 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4840 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4841 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4843 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4844 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4845
4846 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004847 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004848 timestamp_ms += kFrameIntervalMs;
4849 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004850 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004851 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4856 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4857 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4858
4859 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004860 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004861 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004862 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4863 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4864
mflodmancc3d4422017-08-03 08:27:51 -07004865 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004866}
4867
mflodmancc3d4422017-08-03 08:27:51 -07004868TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004869 const int kFrameWidth = 1920;
4870 const int kFrameHeight = 1080;
4871 // 3/4 of 1920.
4872 const int kAdaptedFrameWidth = 1440;
4873 // 3/4 of 1080 rounded down to multiple of 4.
4874 const int kAdaptedFrameHeight = 808;
4875 const int kFramerate = 24;
4876
Henrik Boström381d1092020-05-12 18:49:07 +02004877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004878 DataRate::BitsPerSec(kTargetBitrateBps),
4879 DataRate::BitsPerSec(kTargetBitrateBps),
4880 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004881 // Trigger reconfigure encoder (without resetting the entire instance).
4882 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004883 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004884 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4885 video_encoder_config.number_of_streams = 1;
4886 video_encoder_config.video_stream_factory =
4887 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004888 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004889 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004890 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004891
4892 video_source_.set_adaptation_enabled(true);
4893
4894 video_source_.IncomingCapturedFrame(
4895 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004896 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004897
4898 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004899 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004900 video_source_.IncomingCapturedFrame(
4901 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004902 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004903
mflodmancc3d4422017-08-03 08:27:51 -07004904 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004905}
4906
mflodmancc3d4422017-08-03 08:27:51 -07004907TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004908 const int kFrameWidth = 1280;
4909 const int kFrameHeight = 720;
4910 const int kLowFps = 2;
4911 const int kHighFps = 30;
4912
Henrik Boström381d1092020-05-12 18:49:07 +02004913 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004914 DataRate::BitsPerSec(kTargetBitrateBps),
4915 DataRate::BitsPerSec(kTargetBitrateBps),
4916 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004917
4918 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4919 max_framerate_ = kLowFps;
4920
4921 // Insert 2 seconds of 2fps video.
4922 for (int i = 0; i < kLowFps * 2; ++i) {
4923 video_source_.IncomingCapturedFrame(
4924 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4925 WaitForEncodedFrame(timestamp_ms);
4926 timestamp_ms += 1000 / kLowFps;
4927 }
4928
4929 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004931 DataRate::BitsPerSec(kTargetBitrateBps),
4932 DataRate::BitsPerSec(kTargetBitrateBps),
4933 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004934 video_source_.IncomingCapturedFrame(
4935 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4936 WaitForEncodedFrame(timestamp_ms);
4937 timestamp_ms += 1000 / kLowFps;
4938
4939 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4940
4941 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004942 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004943 const int kFrameIntervalMs = 1000 / kHighFps;
4944 max_framerate_ = kHighFps;
4945 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4946 video_source_.IncomingCapturedFrame(
4947 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4948 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4949 // be dropped if the encoder hans't been updated with the new higher target
4950 // framerate yet, causing it to overshoot the target bitrate and then
4951 // suffering the wrath of the media optimizer.
4952 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4953 timestamp_ms += kFrameIntervalMs;
4954 }
4955
4956 // Don expect correct measurement just yet, but it should be higher than
4957 // before.
4958 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4959
mflodmancc3d4422017-08-03 08:27:51 -07004960 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004961}
4962
mflodmancc3d4422017-08-03 08:27:51 -07004963TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004964 const int kFrameWidth = 1280;
4965 const int kFrameHeight = 720;
4966 const int kTargetBitrateBps = 1000000;
4967
4968 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004969 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02004970 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004971 DataRate::BitsPerSec(kTargetBitrateBps),
4972 DataRate::BitsPerSec(kTargetBitrateBps),
4973 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004974 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004975
4976 // Insert a first video frame, causes another bitrate update.
4977 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4978 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4979 video_source_.IncomingCapturedFrame(
4980 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4981 WaitForEncodedFrame(timestamp_ms);
4982
4983 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02004984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4985 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
4986 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004987
4988 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004989 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004990 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004991
4992 // Bitrate observer should not be called.
4993 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4994 video_source_.IncomingCapturedFrame(
4995 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4996 ExpectDroppedFrame();
4997
mflodmancc3d4422017-08-03 08:27:51 -07004998 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004999}
ilnik6b826ef2017-06-16 06:53:48 -07005000
Niels Möller4db138e2018-04-19 09:04:13 +02005001TEST_F(VideoStreamEncoderTest,
5002 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5003 const int kFrameWidth = 1280;
5004 const int kFrameHeight = 720;
5005 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005007 DataRate::BitsPerSec(kTargetBitrateBps),
5008 DataRate::BitsPerSec(kTargetBitrateBps),
5009 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005010 video_source_.IncomingCapturedFrame(
5011 CreateFrame(1, kFrameWidth, kFrameHeight));
5012 WaitForEncodedFrame(1);
5013 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5014 .low_encode_usage_threshold_percent,
5015 default_options.low_encode_usage_threshold_percent);
5016 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5017 .high_encode_usage_threshold_percent,
5018 default_options.high_encode_usage_threshold_percent);
5019 video_stream_encoder_->Stop();
5020}
5021
5022TEST_F(VideoStreamEncoderTest,
5023 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5024 const int kFrameWidth = 1280;
5025 const int kFrameHeight = 720;
5026 CpuOveruseOptions hardware_options;
5027 hardware_options.low_encode_usage_threshold_percent = 150;
5028 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005029 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005030
Henrik Boström381d1092020-05-12 18:49:07 +02005031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005032 DataRate::BitsPerSec(kTargetBitrateBps),
5033 DataRate::BitsPerSec(kTargetBitrateBps),
5034 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005035 video_source_.IncomingCapturedFrame(
5036 CreateFrame(1, kFrameWidth, kFrameHeight));
5037 WaitForEncodedFrame(1);
5038 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5039 .low_encode_usage_threshold_percent,
5040 hardware_options.low_encode_usage_threshold_percent);
5041 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5042 .high_encode_usage_threshold_percent,
5043 hardware_options.high_encode_usage_threshold_percent);
5044 video_stream_encoder_->Stop();
5045}
5046
Niels Möller6bb5ab92019-01-11 11:11:10 +01005047TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5048 const int kFrameWidth = 320;
5049 const int kFrameHeight = 240;
5050 const int kFps = 30;
5051 const int kTargetBitrateBps = 120000;
5052 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5053
Henrik Boström381d1092020-05-12 18:49:07 +02005054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005055 DataRate::BitsPerSec(kTargetBitrateBps),
5056 DataRate::BitsPerSec(kTargetBitrateBps),
5057 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005058
5059 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5060 max_framerate_ = kFps;
5061
5062 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5063 fake_encoder_.SimulateOvershoot(1.0);
5064 int num_dropped = 0;
5065 for (int i = 0; i < kNumFramesInRun; ++i) {
5066 video_source_.IncomingCapturedFrame(
5067 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5068 // Wait up to two frame durations for a frame to arrive.
5069 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5070 ++num_dropped;
5071 }
5072 timestamp_ms += 1000 / kFps;
5073 }
5074
Erik Språnga8d48ab2019-02-08 14:17:40 +01005075 // Framerate should be measured to be near the expected target rate.
5076 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5077
5078 // Frame drops should be within 5% of expected 0%.
5079 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005080
5081 // Make encoder produce frames at double the expected bitrate during 3 seconds
5082 // of video, verify number of drops. Rate needs to be slightly changed in
5083 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005084 double overshoot_factor = 2.0;
5085 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5086 // With bitrate adjuster, when need to overshoot even more to trigger
5087 // frame dropping.
5088 overshoot_factor *= 2;
5089 }
5090 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005092 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5093 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5094 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005095 num_dropped = 0;
5096 for (int i = 0; i < kNumFramesInRun; ++i) {
5097 video_source_.IncomingCapturedFrame(
5098 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5099 // Wait up to two frame durations for a frame to arrive.
5100 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5101 ++num_dropped;
5102 }
5103 timestamp_ms += 1000 / kFps;
5104 }
5105
Henrik Boström381d1092020-05-12 18:49:07 +02005106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005107 DataRate::BitsPerSec(kTargetBitrateBps),
5108 DataRate::BitsPerSec(kTargetBitrateBps),
5109 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005110
5111 // Target framerate should be still be near the expected target, despite
5112 // the frame drops.
5113 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5114
5115 // Frame drops should be within 5% of expected 50%.
5116 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005117
5118 video_stream_encoder_->Stop();
5119}
5120
5121TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5122 const int kFrameWidth = 320;
5123 const int kFrameHeight = 240;
5124 const int kActualInputFps = 24;
5125 const int kTargetBitrateBps = 120000;
5126
5127 ASSERT_GT(max_framerate_, kActualInputFps);
5128
5129 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5130 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005132 DataRate::BitsPerSec(kTargetBitrateBps),
5133 DataRate::BitsPerSec(kTargetBitrateBps),
5134 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005135
5136 // Insert 3 seconds of video, with an input fps lower than configured max.
5137 for (int i = 0; i < kActualInputFps * 3; ++i) {
5138 video_source_.IncomingCapturedFrame(
5139 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5140 // Wait up to two frame durations for a frame to arrive.
5141 WaitForEncodedFrame(timestamp_ms);
5142 timestamp_ms += 1000 / kActualInputFps;
5143 }
5144
5145 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5146
5147 video_stream_encoder_->Stop();
5148}
5149
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005150TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5151 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005152 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005153 DataRate::BitsPerSec(kTargetBitrateBps),
5154 DataRate::BitsPerSec(kTargetBitrateBps),
5155 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005156
5157 fake_encoder_.BlockNextEncode();
5158 video_source_.IncomingCapturedFrame(
5159 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5160 WaitForEncodedFrame(1);
5161 // On the very first frame full update should be forced.
5162 rect = fake_encoder_.GetLastUpdateRect();
5163 EXPECT_EQ(rect.offset_x, 0);
5164 EXPECT_EQ(rect.offset_y, 0);
5165 EXPECT_EQ(rect.height, codec_height_);
5166 EXPECT_EQ(rect.width, codec_width_);
5167 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5168 // call to ContinueEncode.
5169 video_source_.IncomingCapturedFrame(
5170 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5171 ExpectDroppedFrame();
5172 video_source_.IncomingCapturedFrame(
5173 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5174 ExpectDroppedFrame();
5175 fake_encoder_.ContinueEncode();
5176 WaitForEncodedFrame(3);
5177 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5178 rect = fake_encoder_.GetLastUpdateRect();
5179 EXPECT_EQ(rect.offset_x, 1);
5180 EXPECT_EQ(rect.offset_y, 0);
5181 EXPECT_EQ(rect.width, 10);
5182 EXPECT_EQ(rect.height, 1);
5183
5184 video_source_.IncomingCapturedFrame(
5185 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5186 WaitForEncodedFrame(4);
5187 // Previous frame was encoded, so no accumulation should happen.
5188 rect = fake_encoder_.GetLastUpdateRect();
5189 EXPECT_EQ(rect.offset_x, 0);
5190 EXPECT_EQ(rect.offset_y, 0);
5191 EXPECT_EQ(rect.width, 1);
5192 EXPECT_EQ(rect.height, 1);
5193
5194 video_stream_encoder_->Stop();
5195}
5196
Erik Språngd7329ca2019-02-21 21:19:53 +01005197TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005198 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005199 DataRate::BitsPerSec(kTargetBitrateBps),
5200 DataRate::BitsPerSec(kTargetBitrateBps),
5201 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005202
5203 // First frame is always keyframe.
5204 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5205 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005206 EXPECT_THAT(
5207 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005208 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005209
5210 // Insert delta frame.
5211 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5212 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005213 EXPECT_THAT(
5214 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005215 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005216
5217 // Request next frame be a key-frame.
5218 video_stream_encoder_->SendKeyFrame();
5219 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5220 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005221 EXPECT_THAT(
5222 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005223 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005224
5225 video_stream_encoder_->Stop();
5226}
5227
5228TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5229 // Setup simulcast with three streams.
5230 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005232 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5233 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5234 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005235 // Wait for all three layers before triggering event.
5236 sink_.SetNumExpectedLayers(3);
5237
5238 // First frame is always keyframe.
5239 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5240 WaitForEncodedFrame(1);
5241 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005242 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5243 VideoFrameType::kVideoFrameKey,
5244 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005245
5246 // Insert delta frame.
5247 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5248 WaitForEncodedFrame(2);
5249 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005250 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5251 VideoFrameType::kVideoFrameDelta,
5252 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005253
5254 // Request next frame be a key-frame.
5255 // Only first stream is configured to produce key-frame.
5256 video_stream_encoder_->SendKeyFrame();
5257 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5258 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005259
5260 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5261 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005262 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005263 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005264 VideoFrameType::kVideoFrameKey,
5265 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005266
5267 video_stream_encoder_->Stop();
5268}
5269
5270TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5271 // Configure internal source factory and setup test again.
5272 encoder_factory_.SetHasInternalSource(true);
5273 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005274 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005275 DataRate::BitsPerSec(kTargetBitrateBps),
5276 DataRate::BitsPerSec(kTargetBitrateBps),
5277 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005278
5279 // Call encoder directly, simulating internal source where encoded frame
5280 // callback in VideoStreamEncoder is called despite no OnFrame().
5281 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5282 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005283 EXPECT_THAT(
5284 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005285 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005286
Niels Möller8f7ce222019-03-21 15:43:58 +01005287 const std::vector<VideoFrameType> kDeltaFrame = {
5288 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005289 // Need to set timestamp manually since manually for injected frame.
5290 VideoFrame frame = CreateFrame(101, nullptr);
5291 frame.set_timestamp(101);
5292 fake_encoder_.InjectFrame(frame, false);
5293 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005294 EXPECT_THAT(
5295 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005296 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005297
5298 // Request key-frame. The forces a dummy frame down into the encoder.
5299 fake_encoder_.ExpectNullFrame();
5300 video_stream_encoder_->SendKeyFrame();
5301 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005302 EXPECT_THAT(
5303 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005304 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005305
5306 video_stream_encoder_->Stop();
5307}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005308
5309TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5310 // Configure internal source factory and setup test again.
5311 encoder_factory_.SetHasInternalSource(true);
5312 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005314 DataRate::BitsPerSec(kTargetBitrateBps),
5315 DataRate::BitsPerSec(kTargetBitrateBps),
5316 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005317
5318 int64_t timestamp = 1;
5319 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005320 image.SetEncodedData(
5321 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005322 image.capture_time_ms_ = ++timestamp;
5323 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5324 const int64_t kEncodeFinishDelayMs = 10;
5325 image.timing_.encode_start_ms = timestamp;
5326 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5327 fake_encoder_.InjectEncodedImage(image);
5328 // Wait for frame without incrementing clock.
5329 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5330 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5331 // capture timestamp should be kEncodeFinishDelayMs in the past.
5332 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5333 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5334 kEncodeFinishDelayMs);
5335
5336 video_stream_encoder_->Stop();
5337}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005338
5339TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5340 // Configure internal source factory and setup test again.
5341 encoder_factory_.SetHasInternalSource(true);
5342 ResetEncoder("H264", 1, 1, 1, false);
5343
5344 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5345 image._frameType = VideoFrameType::kVideoFrameKey;
5346
5347 CodecSpecificInfo codec_specific_info;
5348 codec_specific_info.codecType = kVideoCodecH264;
5349
5350 RTPFragmentationHeader fragmentation;
5351 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5352 fragmentation.fragmentationOffset[0] = 4;
5353 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5354
5355 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5356 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5357
5358 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5359 testing::ElementsAreArray(optimal_sps));
5360 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5361 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5362 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5363 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5364
5365 video_stream_encoder_->Stop();
5366}
5367
5368TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5369 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5370 0x00, 0x00, 0x03, 0x03, 0xF4,
5371 0x05, 0x03, 0xC7, 0xC0};
5372
5373 // Configure internal source factory and setup test again.
5374 encoder_factory_.SetHasInternalSource(true);
5375 ResetEncoder("H264", 1, 1, 1, false);
5376
5377 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5378 image._frameType = VideoFrameType::kVideoFrameKey;
5379
5380 CodecSpecificInfo codec_specific_info;
5381 codec_specific_info.codecType = kVideoCodecH264;
5382
5383 RTPFragmentationHeader fragmentation;
5384 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5385 fragmentation.fragmentationOffset[0] = 4;
5386 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5387
5388 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5389 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5390
5391 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5392 testing::ElementsAreArray(optimal_sps));
5393 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5394 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5395 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5396 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5397
5398 video_stream_encoder_->Stop();
5399}
5400
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005401TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5402 const int kFrameWidth = 1280;
5403 const int kFrameHeight = 720;
5404 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5405
Henrik Boström381d1092020-05-12 18:49:07 +02005406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005407 DataRate::BitsPerSec(kTargetBitrateBps),
5408 DataRate::BitsPerSec(kTargetBitrateBps),
5409 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005410 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5411
5412 // Insert a first video frame. It should be dropped because of downscale in
5413 // resolution.
5414 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5415 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5416 frame.set_rotation(kVideoRotation_270);
5417 video_source_.IncomingCapturedFrame(frame);
5418
5419 ExpectDroppedFrame();
5420
5421 // Second frame is downscaled.
5422 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5423 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5424 frame.set_rotation(kVideoRotation_90);
5425 video_source_.IncomingCapturedFrame(frame);
5426
5427 WaitForEncodedFrame(timestamp_ms);
5428 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5429
5430 // Insert another frame, also downscaled.
5431 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5432 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5433 frame.set_rotation(kVideoRotation_180);
5434 video_source_.IncomingCapturedFrame(frame);
5435
5436 WaitForEncodedFrame(timestamp_ms);
5437 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5438
5439 video_stream_encoder_->Stop();
5440}
5441
Erik Språng5056af02019-09-02 15:53:11 +02005442TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5443 const int kFrameWidth = 320;
5444 const int kFrameHeight = 180;
5445
5446 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005448 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5449 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5450 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005451 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005452 /*rtt_ms=*/0,
5453 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005454
5455 // Insert a first video frame so that encoder gets configured.
5456 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5457 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5458 frame.set_rotation(kVideoRotation_270);
5459 video_source_.IncomingCapturedFrame(frame);
5460 WaitForEncodedFrame(timestamp_ms);
5461
5462 // Set a target rate below the minimum allowed by the codec settings.
5463 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005464 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5465 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005467 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005468 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005469 /*link_allocation=*/target_rate,
5470 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005471 /*rtt_ms=*/0,
5472 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005473 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5474
5475 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5476 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5477 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005478 DataRate allocation_sum =
5479 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005480 EXPECT_EQ(min_rate, allocation_sum);
5481 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5482
5483 video_stream_encoder_->Stop();
5484}
5485
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005486TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005488 DataRate::BitsPerSec(kTargetBitrateBps),
5489 DataRate::BitsPerSec(kTargetBitrateBps),
5490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005491 // Capture a frame and wait for it to synchronize with the encoder thread.
5492 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5493 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5494 WaitForEncodedFrame(1);
5495
5496 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5497 ASSERT_TRUE(prev_rate_settings.has_value());
5498 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5499 kDefaultFramerate);
5500
5501 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5502 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5503 timestamp_ms += 1000 / kDefaultFramerate;
5504 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5505 WaitForEncodedFrame(timestamp_ms);
5506 }
5507 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5508 kDefaultFramerate);
5509 // Capture larger frame to trigger a reconfigure.
5510 codec_height_ *= 2;
5511 codec_width_ *= 2;
5512 timestamp_ms += 1000 / kDefaultFramerate;
5513 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5514 WaitForEncodedFrame(timestamp_ms);
5515
5516 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5517 auto current_rate_settings =
5518 fake_encoder_.GetAndResetLastRateControlSettings();
5519 // Ensure we have actually reconfigured twice
5520 // The rate settings should have been set again even though
5521 // they haven't changed.
5522 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005523 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005524
5525 video_stream_encoder_->Stop();
5526}
5527
philipeld9cc8c02019-09-16 14:53:40 +02005528struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005529 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5530 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5531 MOCK_METHOD(void,
5532 RequestEncoderSwitch,
5533 (const webrtc::SdpVideoFormat& format),
5534 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005535};
5536
5537TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5538 constexpr int kDontCare = 100;
5539
5540 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5541 video_send_config_.encoder_settings.encoder_switch_request_callback =
5542 &switch_callback;
5543 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5544 encoder_config.codec_type = kVideoCodecVP8;
5545 webrtc::test::ScopedFieldTrials field_trial(
5546 "WebRTC-NetworkCondition-EncoderSwitch/"
5547 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5548 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5549
5550 // Reset encoder for new configuration to take effect.
5551 ConfigureEncoder(std::move(encoder_config));
5552
5553 // Send one frame to trigger ReconfigureEncoder.
5554 video_source_.IncomingCapturedFrame(
5555 CreateFrame(kDontCare, kDontCare, kDontCare));
5556
5557 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005558 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5559 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005560 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005561 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005562
Henrik Boström381d1092020-05-12 18:49:07 +02005563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005564 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5565 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5566 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005567 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005568 /*rtt_ms=*/0,
5569 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005570
5571 video_stream_encoder_->Stop();
5572}
5573
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005574TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5575 constexpr int kDontCare = 100;
5576
5577 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5578 video_send_config_.encoder_settings.encoder_switch_request_callback =
5579 &switch_callback;
5580 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5581 encoder_config.codec_type = kVideoCodecVP8;
5582 webrtc::test::ScopedFieldTrials field_trial(
5583 "WebRTC-NetworkCondition-EncoderSwitch/"
5584 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5585 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5586
5587 // Reset encoder for new configuration to take effect.
5588 ConfigureEncoder(std::move(encoder_config));
5589
5590 // Send one frame to trigger ReconfigureEncoder.
5591 video_source_.IncomingCapturedFrame(
5592 CreateFrame(kDontCare, kDontCare, kDontCare));
5593
5594 using Config = EncoderSwitchRequestCallback::Config;
5595 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5596 .Times(0);
5597
Henrik Boström381d1092020-05-12 18:49:07 +02005598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005599 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5600 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5601 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5602 /*fraction_lost=*/0,
5603 /*rtt_ms=*/0,
5604 /*cwnd_reduce_ratio=*/0);
5605
5606 video_stream_encoder_->Stop();
5607}
5608
philipeld9cc8c02019-09-16 14:53:40 +02005609TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5610 constexpr int kSufficientBitrateToNotDrop = 1000;
5611 constexpr int kHighRes = 500;
5612 constexpr int kLowRes = 100;
5613
5614 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5615 video_send_config_.encoder_settings.encoder_switch_request_callback =
5616 &switch_callback;
5617 webrtc::test::ScopedFieldTrials field_trial(
5618 "WebRTC-NetworkCondition-EncoderSwitch/"
5619 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5620 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5621 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5622 encoder_config.codec_type = kVideoCodecH264;
5623
5624 // Reset encoder for new configuration to take effect.
5625 ConfigureEncoder(std::move(encoder_config));
5626
5627 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5628 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5629 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005631 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5632 /*stable_target_bitrate=*/
5633 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5634 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005635 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005636 /*rtt_ms=*/0,
5637 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005638
5639 // Send one frame to trigger ReconfigureEncoder.
5640 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5641 WaitForEncodedFrame(1);
5642
5643 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005644 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5645 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005646 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005647 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005648
5649 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5650 WaitForEncodedFrame(2);
5651
5652 video_stream_encoder_->Stop();
5653}
5654
philipel9b058032020-02-10 11:30:00 +01005655TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5656 constexpr int kDontCare = 100;
5657 StrictMock<MockEncoderSelector> encoder_selector;
5658 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5659 &fake_encoder_, &encoder_selector);
5660 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5661
5662 // Reset encoder for new configuration to take effect.
5663 ConfigureEncoder(video_encoder_config_.Copy());
5664
5665 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5666
5667 video_source_.IncomingCapturedFrame(
5668 CreateFrame(kDontCare, kDontCare, kDontCare));
5669 video_stream_encoder_->Stop();
5670
5671 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5672 // to it's factory, so in order for the encoder instance in the
5673 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5674 // reset the |video_stream_encoder_| here.
5675 video_stream_encoder_.reset();
5676}
5677
5678TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5679 constexpr int kDontCare = 100;
5680
5681 NiceMock<MockEncoderSelector> encoder_selector;
5682 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5683 video_send_config_.encoder_settings.encoder_switch_request_callback =
5684 &switch_callback;
5685 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5686 &fake_encoder_, &encoder_selector);
5687 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5688
5689 // Reset encoder for new configuration to take effect.
5690 ConfigureEncoder(video_encoder_config_.Copy());
5691
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005692 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005693 .WillByDefault(Return(SdpVideoFormat("AV1")));
5694 EXPECT_CALL(switch_callback,
5695 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5696 Field(&SdpVideoFormat::name, "AV1"))));
5697
Henrik Boström381d1092020-05-12 18:49:07 +02005698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005699 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5700 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5701 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005702 /*fraction_lost=*/0,
5703 /*rtt_ms=*/0,
5704 /*cwnd_reduce_ratio=*/0);
5705
5706 video_stream_encoder_->Stop();
5707}
5708
5709TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5710 constexpr int kSufficientBitrateToNotDrop = 1000;
5711 constexpr int kDontCare = 100;
5712
5713 NiceMock<MockVideoEncoder> video_encoder;
5714 NiceMock<MockEncoderSelector> encoder_selector;
5715 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5716 video_send_config_.encoder_settings.encoder_switch_request_callback =
5717 &switch_callback;
5718 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5719 &video_encoder, &encoder_selector);
5720 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5721
5722 // Reset encoder for new configuration to take effect.
5723 ConfigureEncoder(video_encoder_config_.Copy());
5724
5725 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5726 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5727 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005729 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5730 /*stable_target_bitrate=*/
5731 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5732 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005733 /*fraction_lost=*/0,
5734 /*rtt_ms=*/0,
5735 /*cwnd_reduce_ratio=*/0);
5736
5737 ON_CALL(video_encoder, Encode(_, _))
5738 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5739 ON_CALL(encoder_selector, OnEncoderBroken())
5740 .WillByDefault(Return(SdpVideoFormat("AV2")));
5741
5742 rtc::Event encode_attempted;
5743 EXPECT_CALL(switch_callback,
5744 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5745 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5746 EXPECT_EQ(format.name, "AV2");
5747 encode_attempted.Set();
5748 });
5749
5750 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5751 encode_attempted.Wait(3000);
5752
5753 video_stream_encoder_->Stop();
5754
5755 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5756 // to it's factory, so in order for the encoder instance in the
5757 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5758 // reset the |video_stream_encoder_| here.
5759 video_stream_encoder_.reset();
5760}
5761
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005762TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005763 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005764 const int kFrameWidth = 320;
5765 const int kFrameHeight = 180;
5766
5767 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005768 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005770 /*target_bitrate=*/rate,
5771 /*stable_target_bitrate=*/rate,
5772 /*link_allocation=*/rate,
5773 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005774 /*rtt_ms=*/0,
5775 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005776
5777 // Insert a first video frame so that encoder gets configured.
5778 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5779 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5780 frame.set_rotation(kVideoRotation_270);
5781 video_source_.IncomingCapturedFrame(frame);
5782 WaitForEncodedFrame(timestamp_ms);
5783 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5784
5785 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005786 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005787 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005788 /*target_bitrate=*/new_stable_rate,
5789 /*stable_target_bitrate=*/new_stable_rate,
5790 /*link_allocation=*/rate,
5791 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005792 /*rtt_ms=*/0,
5793 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005794 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5795 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5796 video_stream_encoder_->Stop();
5797}
5798
5799TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005800 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005801 const int kFrameWidth = 320;
5802 const int kFrameHeight = 180;
5803
5804 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005805 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005807 /*target_bitrate=*/rate,
5808 /*stable_target_bitrate=*/rate,
5809 /*link_allocation=*/rate,
5810 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005811 /*rtt_ms=*/0,
5812 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005813
5814 // Insert a first video frame so that encoder gets configured.
5815 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5816 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5817 frame.set_rotation(kVideoRotation_270);
5818 video_source_.IncomingCapturedFrame(frame);
5819 WaitForEncodedFrame(timestamp_ms);
5820 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5821
5822 // Set a higher target rate without changing the link_allocation. Should not
5823 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005824 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005826 /*target_bitrate=*/rate,
5827 /*stable_target_bitrate=*/new_stable_rate,
5828 /*link_allocation=*/rate,
5829 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005830 /*rtt_ms=*/0,
5831 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005832 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5833 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5834 video_stream_encoder_->Stop();
5835}
5836
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005837TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5838 test::ScopedFieldTrials field_trials(
5839 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5840 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5841 const int kFramerateFps = 30;
5842 const int kWidth = 1920;
5843 const int kHeight = 1080;
5844 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5845 // Works on screenshare mode.
5846 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5847 // We rely on the automatic resolution adaptation, but we handle framerate
5848 // adaptation manually by mocking the stats proxy.
5849 video_source_.set_adaptation_enabled(true);
5850
5851 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005853 DataRate::BitsPerSec(kTargetBitrateBps),
5854 DataRate::BitsPerSec(kTargetBitrateBps),
5855 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005856 video_stream_encoder_->SetSource(&video_source_,
5857 webrtc::DegradationPreference::BALANCED);
5858 VerifyNoLimitation(video_source_.sink_wants());
5859
5860 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5861 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5862
5863 // Pass enough frames with the full update to trigger animation detection.
5864 for (int i = 0; i < kNumFrames; ++i) {
5865 int64_t timestamp_ms =
5866 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5867 frame.set_ntp_time_ms(timestamp_ms);
5868 frame.set_timestamp_us(timestamp_ms * 1000);
5869 video_source_.IncomingCapturedFrame(frame);
5870 WaitForEncodedFrame(timestamp_ms);
5871 }
5872
5873 // Resolution should be limited.
5874 rtc::VideoSinkWants expected;
5875 expected.max_framerate_fps = kFramerateFps;
5876 expected.max_pixel_count = 1280 * 720 + 1;
5877 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5878
5879 // Pass one frame with no known update.
5880 // Resolution cap should be removed immediately.
5881 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5882 frame.set_ntp_time_ms(timestamp_ms);
5883 frame.set_timestamp_us(timestamp_ms * 1000);
5884 frame.clear_update_rect();
5885
5886 video_source_.IncomingCapturedFrame(frame);
5887 WaitForEncodedFrame(timestamp_ms);
5888
5889 // Resolution should be unlimited now.
5890 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5891
5892 video_stream_encoder_->Stop();
5893}
5894
perkj26091b12016-09-01 01:17:40 -07005895} // namespace webrtc