blob: 5123d45bdd23b9d39a5f7669c0c070cf8409c466 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020040#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "system_wrappers/include/sleep.h"
42#include "test/encoder_settings.h"
43#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020044#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010045#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/gmock.h"
47#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020048#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020049#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070050
51namespace webrtc {
52
sprang57c2fff2017-01-16 06:24:02 -080053using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020054using ::testing::AllOf;
55using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010056using ::testing::Matcher;
57using ::testing::NiceMock;
58using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080060
perkj803d97f2016-11-01 11:45:46 -070061namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020062const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010063const int kQpLow = 1;
64const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int kMinFramerateFps = 2;
66const int kMinBalancedFramerateFps = 7;
67const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080068const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010069const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020070const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010071const uint32_t kSimulcastTargetBitrateBps = 3150000;
72const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080073const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070074const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020076const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020077const VideoEncoder::ResolutionBitrateLimits
78 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
79const VideoEncoder::ResolutionBitrateLimits
80 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080081
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020082uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
83 0x00, 0x00, 0x03, 0x03, 0xF4,
84 0x05, 0x03, 0xC7, 0xE0, 0x1B,
85 0x41, 0x10, 0x8D, 0x00};
86
perkj803d97f2016-11-01 11:45:46 -070087class TestBuffer : public webrtc::I420Buffer {
88 public:
89 TestBuffer(rtc::Event* event, int width, int height)
90 : I420Buffer(width, height), event_(event) {}
91
92 private:
93 friend class rtc::RefCountedObject<TestBuffer>;
94 ~TestBuffer() override {
95 if (event_)
96 event_->Set();
97 }
98 rtc::Event* const event_;
99};
100
Noah Richards51db4212019-06-12 06:59:12 -0700101// A fake native buffer that can't be converted to I420.
102class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
103 public:
104 FakeNativeBuffer(rtc::Event* event, int width, int height)
105 : event_(event), width_(width), height_(height) {}
106 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
107 int width() const override { return width_; }
108 int height() const override { return height_; }
109 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
110 return nullptr;
111 }
112
113 private:
114 friend class rtc::RefCountedObject<FakeNativeBuffer>;
115 ~FakeNativeBuffer() override {
116 if (event_)
117 event_->Set();
118 }
119 rtc::Event* const event_;
120 const int width_;
121 const int height_;
122};
123
Niels Möller7dc26b72017-12-06 10:27:48 +0100124class CpuOveruseDetectorProxy : public OveruseFrameDetector {
125 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200126 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
127 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200128 last_target_framerate_fps_(-1),
129 framerate_updated_event_(true /* manual_reset */,
130 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100131 virtual ~CpuOveruseDetectorProxy() {}
132
133 void OnTargetFramerateUpdated(int framerate_fps) override {
134 rtc::CritScope cs(&lock_);
135 last_target_framerate_fps_ = framerate_fps;
136 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200137 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100138 }
139
140 int GetLastTargetFramerate() {
141 rtc::CritScope cs(&lock_);
142 return last_target_framerate_fps_;
143 }
144
Niels Möller4db138e2018-04-19 09:04:13 +0200145 CpuOveruseOptions GetOptions() { return options_; }
146
Henrik Boström381d1092020-05-12 18:49:07 +0200147 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
148
Niels Möller7dc26b72017-12-06 10:27:48 +0100149 private:
150 rtc::CriticalSection lock_;
151 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200152 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100153};
154
Henrik Boström91aa7322020-04-28 12:24:33 +0200155class FakeQualityScalerQpUsageHandlerCallback
156 : public QualityScalerQpUsageHandlerCallbackInterface {
157 public:
158 FakeQualityScalerQpUsageHandlerCallback()
159 : QualityScalerQpUsageHandlerCallbackInterface() {}
160 ~FakeQualityScalerQpUsageHandlerCallback() override {}
161
162 void OnQpUsageHandled(bool clear_qp_samples) override {
163 clear_qp_samples_result_ = clear_qp_samples;
164 }
165
166 absl::optional<bool> clear_qp_samples_result() const {
167 return clear_qp_samples_result_;
168 }
169
170 private:
171 absl::optional<bool> clear_qp_samples_result_;
172};
173
Henrik Boström381d1092020-05-12 18:49:07 +0200174class VideoSourceRestrictionsUpdatedListener
175 : public ResourceAdaptationProcessorListener {
176 public:
177 VideoSourceRestrictionsUpdatedListener()
178 : was_restrictions_updated_(false), restrictions_updated_event_() {}
179 ~VideoSourceRestrictionsUpdatedListener() override {
180 RTC_DCHECK(was_restrictions_updated_);
181 }
182
183 rtc::Event* restrictions_updated_event() {
184 return &restrictions_updated_event_;
185 }
186
187 // ResourceAdaptationProcessorListener implementation.
188 void OnVideoSourceRestrictionsUpdated(
189 VideoSourceRestrictions restrictions,
190 const VideoAdaptationCounters& adaptation_counters,
191 rtc::scoped_refptr<Resource> reason) override {
192 was_restrictions_updated_ = true;
193 restrictions_updated_event_.Set();
194 }
195
196 private:
197 bool was_restrictions_updated_;
198 rtc::Event restrictions_updated_event_;
199};
200
mflodmancc3d4422017-08-03 08:27:51 -0700201class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700202 public:
Niels Möller213618e2018-07-24 09:29:58 +0200203 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200204 const VideoStreamEncoderSettings& settings,
205 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100206 : VideoStreamEncoder(Clock::GetRealTimeClock(),
207 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200208 stats_proxy,
209 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200210 std::unique_ptr<OveruseFrameDetector>(
211 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100212 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100213 task_queue_factory),
Henrik Boströmc55516d2020-05-11 16:29:22 +0200214 fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
215 fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
Henrik Boström381d1092020-05-12 18:49:07 +0200216 fake_cpu_resource_->Initialize(encoder_queue(),
217 resource_adaptation_queue());
218 fake_quality_resource_->Initialize(encoder_queue(),
219 resource_adaptation_queue());
Henrik Boströmc55516d2020-05-11 16:29:22 +0200220 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200221 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200222 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100223 }
perkj803d97f2016-11-01 11:45:46 -0700224
Henrik Boström381d1092020-05-12 18:49:07 +0200225 void SetSourceAndWaitForRestrictionsUpdated(
226 rtc::VideoSourceInterface<VideoFrame>* source,
227 const DegradationPreference& degradation_preference) {
228 VideoSourceRestrictionsUpdatedListener listener;
229 AddAdaptationListenerForTesting(&listener);
230 SetSource(source, degradation_preference);
231 listener.restrictions_updated_event()->Wait(5000);
232 RemoveAdaptationListenerForTesting(&listener);
233 }
234
235 void SetSourceAndWaitForFramerateUpdated(
236 rtc::VideoSourceInterface<VideoFrame>* source,
237 const DegradationPreference& degradation_preference) {
238 overuse_detector_proxy_->framerate_updated_event()->Reset();
239 SetSource(source, degradation_preference);
240 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
241 }
242
243 void OnBitrateUpdatedAndWaitForManagedResources(
244 DataRate target_bitrate,
245 DataRate stable_target_bitrate,
246 DataRate link_allocation,
247 uint8_t fraction_lost,
248 int64_t round_trip_time_ms,
249 double cwnd_reduce_ratio) {
250 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
251 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
252 // Bitrate is updated on the encoder queue.
253 WaitUntilTaskQueueIsIdle();
254 // Give the managed resources time to react to the new bitrate.
255 // TODO(hbos): Can we await an appropriate event instead?
256 WaitUntilAdaptationTaskQueueIsIdle();
257 }
258
259 void WaitUntilAdaptationTaskQueueIsIdle() {
260 rtc::Event event;
261 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
262 ASSERT_TRUE(event.Wait(5000));
263 }
264
kthelgason2fc52542017-03-03 00:24:41 -0800265 // This is used as a synchronisation mechanism, to make sure that the
266 // encoder queue is not blocked before we start sending it frames.
267 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100268 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200269 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800270 ASSERT_TRUE(event.Wait(5000));
271 }
272
Henrik Boström91aa7322020-04-28 12:24:33 +0200273 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200274 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200275 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200276 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200277 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
278 event.Set();
279 });
280 ASSERT_TRUE(event.Wait(5000));
281 }
282 void TriggerCpuUnderuse() {
283 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200284 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200285 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
286 event.Set();
287 });
288 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200289 }
kthelgason876222f2016-11-29 01:44:11 -0800290
Henrik Boström91aa7322020-04-28 12:24:33 +0200291 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200292 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200293 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200294 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200295 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
296 event.Set();
297 });
298 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200299 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200300 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200301 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200302 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200303 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
304 event.Set();
305 });
306 ASSERT_TRUE(event.Wait(5000));
307 }
308
309 // Fakes high QP resource usage measurements on the real
310 // QualityScalerResource. Returns whether or not QP samples would have been
311 // cleared if this had been a real signal from the QualityScaler.
312 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
313 rtc::Event event;
314 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
315 new FakeQualityScalerQpUsageHandlerCallback();
316 encoder_queue()->PostTask([this, &event, callback] {
Henrik Boström381d1092020-05-12 18:49:07 +0200317 // This should post a usage measurement to the adaptation processor.
Henrik Boström91aa7322020-04-28 12:24:33 +0200318 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
Henrik Boström381d1092020-05-12 18:49:07 +0200319 // Give the processor a chance to react and trigger adaptation on the
320 // adaptation queue.
321 resource_adaptation_queue()->PostTask([this, &event] {
322 // Finally, give the QualityScalerResource time to resolve the callback
323 // on the encoder queue.
324 encoder_queue()->PostTask([&event] { event.Set(); });
325 });
Henrik Boström91aa7322020-04-28 12:24:33 +0200326 });
327 EXPECT_TRUE(event.Wait(5000));
328 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
329 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200330 }
sprangfda496a2017-06-15 04:21:07 -0700331
Niels Möller7dc26b72017-12-06 10:27:48 +0100332 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200333 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
334 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700335};
336
asapersson5f7226f2016-11-25 04:37:00 -0800337class VideoStreamFactory
338 : public VideoEncoderConfig::VideoStreamFactoryInterface {
339 public:
sprangfda496a2017-06-15 04:21:07 -0700340 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
341 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800342 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700343 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800344 }
345
346 private:
347 std::vector<VideoStream> CreateEncoderStreams(
348 int width,
349 int height,
350 const VideoEncoderConfig& encoder_config) override {
351 std::vector<VideoStream> streams =
352 test::CreateVideoStreams(width, height, encoder_config);
353 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100354 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700355 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800356 }
357 return streams;
358 }
sprangfda496a2017-06-15 04:21:07 -0700359
asapersson5f7226f2016-11-25 04:37:00 -0800360 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700361 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800362};
363
Noah Richards51db4212019-06-12 06:59:12 -0700364// Simulates simulcast behavior and makes highest stream resolutions divisible
365// by 4.
366class CroppingVideoStreamFactory
367 : public VideoEncoderConfig::VideoStreamFactoryInterface {
368 public:
369 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
370 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
371 EXPECT_GT(num_temporal_layers, 0u);
372 EXPECT_GT(framerate, 0);
373 }
374
375 private:
376 std::vector<VideoStream> CreateEncoderStreams(
377 int width,
378 int height,
379 const VideoEncoderConfig& encoder_config) override {
380 std::vector<VideoStream> streams = test::CreateVideoStreams(
381 width - width % 4, height - height % 4, encoder_config);
382 for (VideoStream& stream : streams) {
383 stream.num_temporal_layers = num_temporal_layers_;
384 stream.max_framerate = framerate_;
385 }
386 return streams;
387 }
388
389 const size_t num_temporal_layers_;
390 const int framerate_;
391};
392
sprangb1ca0732017-02-01 08:38:12 -0800393class AdaptingFrameForwarder : public test::FrameForwarder {
394 public:
395 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700396 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800397
398 void set_adaptation_enabled(bool enabled) {
399 rtc::CritScope cs(&crit_);
400 adaptation_enabled_ = enabled;
401 }
402
asaperssonfab67072017-04-04 05:51:49 -0700403 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800404 rtc::CritScope cs(&crit_);
405 return adaptation_enabled_;
406 }
407
asapersson09f05612017-05-15 23:40:18 -0700408 rtc::VideoSinkWants last_wants() const {
409 rtc::CritScope cs(&crit_);
410 return last_wants_;
411 }
412
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200413 absl::optional<int> last_sent_width() const { return last_width_; }
414 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800415
sprangb1ca0732017-02-01 08:38:12 -0800416 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
417 int cropped_width = 0;
418 int cropped_height = 0;
419 int out_width = 0;
420 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700421 if (adaption_enabled()) {
422 if (adapter_.AdaptFrameResolution(
423 video_frame.width(), video_frame.height(),
424 video_frame.timestamp_us() * 1000, &cropped_width,
425 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100426 VideoFrame adapted_frame =
427 VideoFrame::Builder()
428 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
429 nullptr, out_width, out_height))
430 .set_timestamp_rtp(99)
431 .set_timestamp_ms(99)
432 .set_rotation(kVideoRotation_0)
433 .build();
sprangc5d62e22017-04-02 23:53:04 -0700434 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100435 if (video_frame.has_update_rect()) {
436 adapted_frame.set_update_rect(
437 video_frame.update_rect().ScaleWithFrame(
438 video_frame.width(), video_frame.height(), 0, 0,
439 video_frame.width(), video_frame.height(), out_width,
440 out_height));
441 }
sprangc5d62e22017-04-02 23:53:04 -0700442 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800443 last_width_.emplace(adapted_frame.width());
444 last_height_.emplace(adapted_frame.height());
445 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200446 last_width_ = absl::nullopt;
447 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700448 }
sprangb1ca0732017-02-01 08:38:12 -0800449 } else {
450 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800451 last_width_.emplace(video_frame.width());
452 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800453 }
454 }
455
456 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
457 const rtc::VideoSinkWants& wants) override {
458 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700459 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100460 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800461 test::FrameForwarder::AddOrUpdateSink(sink, wants);
462 }
sprangb1ca0732017-02-01 08:38:12 -0800463 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700464 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
465 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200466 absl::optional<int> last_width_;
467 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800468};
sprangc5d62e22017-04-02 23:53:04 -0700469
Niels Möller213618e2018-07-24 09:29:58 +0200470// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700471class MockableSendStatisticsProxy : public SendStatisticsProxy {
472 public:
473 MockableSendStatisticsProxy(Clock* clock,
474 const VideoSendStream::Config& config,
475 VideoEncoderConfig::ContentType content_type)
476 : SendStatisticsProxy(clock, config, content_type) {}
477
478 VideoSendStream::Stats GetStats() override {
479 rtc::CritScope cs(&lock_);
480 if (mock_stats_)
481 return *mock_stats_;
482 return SendStatisticsProxy::GetStats();
483 }
484
Niels Möller213618e2018-07-24 09:29:58 +0200485 int GetInputFrameRate() const override {
486 rtc::CritScope cs(&lock_);
487 if (mock_stats_)
488 return mock_stats_->input_frame_rate;
489 return SendStatisticsProxy::GetInputFrameRate();
490 }
sprangc5d62e22017-04-02 23:53:04 -0700491 void SetMockStats(const VideoSendStream::Stats& stats) {
492 rtc::CritScope cs(&lock_);
493 mock_stats_.emplace(stats);
494 }
495
496 void ResetMockStats() {
497 rtc::CritScope cs(&lock_);
498 mock_stats_.reset();
499 }
500
501 private:
502 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200503 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700504};
505
sprang4847ae62017-06-27 07:06:52 -0700506class MockBitrateObserver : public VideoBitrateAllocationObserver {
507 public:
Erik Språng566124a2018-04-23 12:32:22 +0200508 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700509};
510
philipel9b058032020-02-10 11:30:00 +0100511class MockEncoderSelector
512 : public VideoEncoderFactory::EncoderSelectorInterface {
513 public:
514 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
Mirta Dvornicic4f34d782020-02-26 13:01:19 +0100515 MOCK_METHOD1(OnAvailableBitrate,
philipel9b058032020-02-10 11:30:00 +0100516 absl::optional<SdpVideoFormat>(const DataRate& rate));
517 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
518};
519
perkj803d97f2016-11-01 11:45:46 -0700520} // namespace
521
mflodmancc3d4422017-08-03 08:27:51 -0700522class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700523 public:
524 static const int kDefaultTimeoutMs = 30 * 1000;
525
mflodmancc3d4422017-08-03 08:27:51 -0700526 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700527 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700528 codec_width_(320),
529 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200530 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200531 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700532 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200533 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700534 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700535 Clock::GetRealTimeClock(),
536 video_send_config_,
537 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700538 sink_(&fake_encoder_) {}
539
540 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700541 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700542 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200543 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800544 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200545 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200546 video_send_config_.rtp.payload_name = "FAKE";
547 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700548
Per512ecb32016-09-23 15:52:06 +0200549 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200550 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700551 video_encoder_config.video_stream_factory =
552 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100553 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700554
555 // Framerate limit is specified by the VideoStreamFactory.
556 std::vector<VideoStream> streams =
557 video_encoder_config.video_stream_factory->CreateEncoderStreams(
558 codec_width_, codec_height_, video_encoder_config);
559 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100560 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700561
Niels Möllerf1338562018-04-26 09:51:47 +0200562 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800563 }
564
Niels Möllerf1338562018-04-26 09:51:47 +0200565 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700566 if (video_stream_encoder_)
567 video_stream_encoder_->Stop();
568 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200569 stats_proxy_.get(), video_send_config_.encoder_settings,
570 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700571 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
572 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700573 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700574 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
575 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200576 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700577 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800578 }
579
580 void ResetEncoder(const std::string& payload_name,
581 size_t num_streams,
582 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700583 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700584 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200585 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800586
587 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200588 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800589 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100590 video_encoder_config.max_bitrate_bps =
591 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800592 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700593 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
594 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700595 video_encoder_config.content_type =
596 screenshare ? VideoEncoderConfig::ContentType::kScreen
597 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700598 if (payload_name == "VP9") {
599 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
600 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
601 video_encoder_config.encoder_specific_settings =
602 new rtc::RefCountedObject<
603 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
604 }
Niels Möllerf1338562018-04-26 09:51:47 +0200605 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700606 }
607
sprang57c2fff2017-01-16 06:24:02 -0800608 VideoFrame CreateFrame(int64_t ntp_time_ms,
609 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100610 VideoFrame frame =
611 VideoFrame::Builder()
612 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
613 destruction_event, codec_width_, codec_height_))
614 .set_timestamp_rtp(99)
615 .set_timestamp_ms(99)
616 .set_rotation(kVideoRotation_0)
617 .build();
sprang57c2fff2017-01-16 06:24:02 -0800618 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700619 return frame;
620 }
621
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100622 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
623 rtc::Event* destruction_event,
624 int offset_x) const {
625 VideoFrame frame =
626 VideoFrame::Builder()
627 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
628 destruction_event, codec_width_, codec_height_))
629 .set_timestamp_rtp(99)
630 .set_timestamp_ms(99)
631 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100632 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100633 .build();
634 frame.set_ntp_time_ms(ntp_time_ms);
635 return frame;
636 }
637
sprang57c2fff2017-01-16 06:24:02 -0800638 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100639 VideoFrame frame =
640 VideoFrame::Builder()
641 .set_video_frame_buffer(
642 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
643 .set_timestamp_rtp(99)
644 .set_timestamp_ms(99)
645 .set_rotation(kVideoRotation_0)
646 .build();
sprang57c2fff2017-01-16 06:24:02 -0800647 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700648 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700649 return frame;
650 }
651
Noah Richards51db4212019-06-12 06:59:12 -0700652 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
653 rtc::Event* destruction_event,
654 int width,
655 int height) const {
656 VideoFrame frame =
657 VideoFrame::Builder()
658 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
659 destruction_event, width, height))
660 .set_timestamp_rtp(99)
661 .set_timestamp_ms(99)
662 .set_rotation(kVideoRotation_0)
663 .build();
664 frame.set_ntp_time_ms(ntp_time_ms);
665 return frame;
666 }
667
668 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
669 rtc::Event* destruction_event) const {
670 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
671 codec_height_);
672 }
673
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100674 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
675 MockBitrateObserver bitrate_observer;
676 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
677
678 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
679 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200680 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100681 DataRate::BitsPerSec(kTargetBitrateBps),
682 DataRate::BitsPerSec(kTargetBitrateBps),
683 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100684
685 video_source_.IncomingCapturedFrame(
686 CreateFrame(1, codec_width_, codec_height_));
687 WaitForEncodedFrame(1);
688 }
689
asapersson02465b82017-04-10 01:12:52 -0700690 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700691 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700692 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
693 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700694 }
695
asapersson09f05612017-05-15 23:40:18 -0700696 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
697 const rtc::VideoSinkWants& wants2) {
698 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
699 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
700 }
701
Åsa Persson8c1bf952018-09-13 10:42:19 +0200702 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
703 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
704 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
705 EXPECT_FALSE(wants.target_pixel_count);
706 }
707
asapersson09f05612017-05-15 23:40:18 -0700708 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
709 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200710 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700711 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
712 EXPECT_GT(wants1.max_pixel_count, 0);
713 }
714
715 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
716 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200717 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700718 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
719 }
720
asaperssonf7e294d2017-06-13 23:25:22 -0700721 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
722 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200723 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700724 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
725 }
726
727 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
728 const rtc::VideoSinkWants& wants2) {
729 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
730 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
731 }
732
733 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
734 const rtc::VideoSinkWants& wants2) {
735 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
736 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
737 }
738
739 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
740 const rtc::VideoSinkWants& wants2) {
741 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
742 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
743 EXPECT_GT(wants1.max_pixel_count, 0);
744 }
745
746 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
747 const rtc::VideoSinkWants& wants2) {
748 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
749 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
750 }
751
asapersson09f05612017-05-15 23:40:18 -0700752 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
753 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200754 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700755 EXPECT_LT(wants.max_pixel_count, pixel_count);
756 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700757 }
758
759 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
760 EXPECT_LT(wants.max_framerate_fps, fps);
761 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
762 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700763 }
764
asaperssonf7e294d2017-06-13 23:25:22 -0700765 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
766 int expected_fps) {
767 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
768 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
769 EXPECT_FALSE(wants.target_pixel_count);
770 }
771
Jonathan Yubc771b72017-12-08 17:04:29 -0800772 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
773 int last_frame_pixels) {
774 // Balanced mode should always scale FPS to the desired range before
775 // attempting to scale resolution.
776 int fps_limit = wants.max_framerate_fps;
777 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100778 EXPECT_LE(7, fps_limit);
779 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800780 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100781 EXPECT_LE(10, fps_limit);
782 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800783 } else if (last_frame_pixels <= 640 * 480) {
784 EXPECT_LE(15, fps_limit);
785 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200786 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800787 }
788 }
789
sprang4847ae62017-06-27 07:06:52 -0700790 void WaitForEncodedFrame(int64_t expected_ntp_time) {
791 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100792 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700793 }
794
795 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
796 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100797 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700798 return ok;
799 }
800
801 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
802 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100803 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700804 }
805
806 void ExpectDroppedFrame() {
807 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100808 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700809 }
810
811 bool WaitForFrame(int64_t timeout_ms) {
812 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100813 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 return ok;
815 }
816
perkj26091b12016-09-01 01:17:40 -0700817 class TestEncoder : public test::FakeEncoder {
818 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100819 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700820
asaperssonfab67072017-04-04 05:51:49 -0700821 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800822 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700823 return config_;
824 }
825
826 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800827 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700828 block_next_encode_ = true;
829 }
830
Erik Språngaed30702018-11-05 12:57:17 +0100831 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800832 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100833 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100834 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100835 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100836 info.scaling_settings = VideoEncoder::ScalingSettings(
837 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100838 }
839 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100840 for (int i = 0; i < kMaxSpatialLayers; ++i) {
841 if (temporal_layers_supported_[i]) {
842 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
843 info.fps_allocation[i].resize(num_layers);
844 }
845 }
Erik Språngaed30702018-11-05 12:57:17 +0100846 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200847
848 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100849 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100850 return info;
kthelgason876222f2016-11-29 01:44:11 -0800851 }
852
Erik Språngb7cb7b52019-02-26 15:52:33 +0100853 int32_t RegisterEncodeCompleteCallback(
854 EncodedImageCallback* callback) override {
855 rtc::CritScope lock(&local_crit_sect_);
856 encoded_image_callback_ = callback;
857 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
858 }
859
perkjfa10b552016-10-02 23:45:26 -0700860 void ContinueEncode() { continue_encode_event_.Set(); }
861
862 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
863 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800864 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700865 EXPECT_EQ(timestamp_, timestamp);
866 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
867 }
868
kthelgason2fc52542017-03-03 00:24:41 -0800869 void SetQualityScaling(bool b) {
870 rtc::CritScope lock(&local_crit_sect_);
871 quality_scaling_ = b;
872 }
kthelgasonad9010c2017-02-14 00:46:51 -0800873
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100874 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
875 rtc::CritScope lock(&local_crit_sect_);
876 requested_resolution_alignment_ = requested_resolution_alignment;
877 }
878
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100879 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
880 rtc::CritScope lock(&local_crit_sect_);
881 is_hardware_accelerated_ = is_hardware_accelerated;
882 }
883
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100884 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
885 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
886 rtc::CritScope lock(&local_crit_sect_);
887 temporal_layers_supported_[spatial_idx] = supported;
888 }
889
Sergey Silkin6456e352019-07-08 17:56:40 +0200890 void SetResolutionBitrateLimits(
891 std::vector<ResolutionBitrateLimits> thresholds) {
892 rtc::CritScope cs(&local_crit_sect_);
893 resolution_bitrate_limits_ = thresholds;
894 }
895
sprangfe627f32017-03-29 08:24:59 -0700896 void ForceInitEncodeFailure(bool force_failure) {
897 rtc::CritScope lock(&local_crit_sect_);
898 force_init_encode_failed_ = force_failure;
899 }
900
Niels Möller6bb5ab92019-01-11 11:11:10 +0100901 void SimulateOvershoot(double rate_factor) {
902 rtc::CritScope lock(&local_crit_sect_);
903 rate_factor_ = rate_factor;
904 }
905
Erik Språngd7329ca2019-02-21 21:19:53 +0100906 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100907 rtc::CritScope lock(&local_crit_sect_);
908 return last_framerate_;
909 }
910
Erik Språngd7329ca2019-02-21 21:19:53 +0100911 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100912 rtc::CritScope lock(&local_crit_sect_);
913 return last_update_rect_;
914 }
915
Niels Möller87e2d782019-03-07 10:18:23 +0100916 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100917 rtc::CritScope lock(&local_crit_sect_);
918 return last_frame_types_;
919 }
920
921 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100922 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100923 keyframe ? VideoFrameType::kVideoFrameKey
924 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100925 {
926 rtc::CritScope lock(&local_crit_sect_);
927 last_frame_types_ = frame_type;
928 }
Niels Möllerb859b322019-03-07 12:40:01 +0100929 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100930 }
931
Erik Språngb7cb7b52019-02-26 15:52:33 +0100932 void InjectEncodedImage(const EncodedImage& image) {
933 rtc::CritScope lock(&local_crit_sect_);
934 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
935 }
936
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200937 void InjectEncodedImage(const EncodedImage& image,
938 const CodecSpecificInfo* codec_specific_info,
939 const RTPFragmentationHeader* fragmentation) {
940 rtc::CritScope lock(&local_crit_sect_);
941 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
942 fragmentation);
943 }
944
Erik Språngd7329ca2019-02-21 21:19:53 +0100945 void ExpectNullFrame() {
946 rtc::CritScope lock(&local_crit_sect_);
947 expect_null_frame_ = true;
948 }
949
Erik Språng5056af02019-09-02 15:53:11 +0200950 absl::optional<VideoEncoder::RateControlParameters>
951 GetAndResetLastRateControlSettings() {
952 auto settings = last_rate_control_settings_;
953 last_rate_control_settings_.reset();
954 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100955 }
956
Sergey Silkin5ee69672019-07-02 14:18:34 +0200957 int GetNumEncoderInitializations() const {
958 rtc::CritScope lock(&local_crit_sect_);
959 return num_encoder_initializations_;
960 }
961
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200962 int GetNumSetRates() const {
963 rtc::CritScope lock(&local_crit_sect_);
964 return num_set_rates_;
965 }
966
perkjfa10b552016-10-02 23:45:26 -0700967 private:
perkj26091b12016-09-01 01:17:40 -0700968 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100969 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700970 bool block_encode;
971 {
brandtre78d2662017-01-16 05:57:16 -0800972 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100973 if (expect_null_frame_) {
974 EXPECT_EQ(input_image.timestamp(), 0u);
975 EXPECT_EQ(input_image.width(), 1);
976 last_frame_types_ = *frame_types;
977 expect_null_frame_ = false;
978 } else {
979 EXPECT_GT(input_image.timestamp(), timestamp_);
980 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
981 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
982 }
perkj26091b12016-09-01 01:17:40 -0700983
984 timestamp_ = input_image.timestamp();
985 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700986 last_input_width_ = input_image.width();
987 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700988 block_encode = block_next_encode_;
989 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100990 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100991 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700992 }
Niels Möllerb859b322019-03-07 12:40:01 +0100993 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700994 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700995 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700996 return result;
997 }
998
sprangfe627f32017-03-29 08:24:59 -0700999 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001000 const Settings& settings) override {
1001 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001002
sprangfe627f32017-03-29 08:24:59 -07001003 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001004 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001005
1006 ++num_encoder_initializations_;
1007
Erik Språng82fad3d2018-03-21 09:57:23 +01001008 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001009 // Simulate setting up temporal layers, in order to validate the life
1010 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001011 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001012 frame_buffer_controller_ =
1013 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001014 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001015 if (force_init_encode_failed_) {
1016 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001017 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001018 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001019
Erik Språngb7cb7b52019-02-26 15:52:33 +01001020 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001021 return res;
1022 }
1023
Erik Språngb7cb7b52019-02-26 15:52:33 +01001024 int32_t Release() override {
1025 rtc::CritScope lock(&local_crit_sect_);
1026 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1027 initialized_ = EncoderState::kUninitialized;
1028 return FakeEncoder::Release();
1029 }
1030
Erik Språng16cb8f52019-04-12 13:59:09 +02001031 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001032 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001033 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001034 VideoBitrateAllocation adjusted_rate_allocation;
1035 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1036 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001037 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001038 adjusted_rate_allocation.SetBitrate(
1039 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001040 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001041 rate_factor_));
1042 }
1043 }
1044 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001045 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001046 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001047 RateControlParameters adjusted_paramters = parameters;
1048 adjusted_paramters.bitrate = adjusted_rate_allocation;
1049 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001050 }
1051
brandtre78d2662017-01-16 05:57:16 -08001052 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001053 enum class EncoderState {
1054 kUninitialized,
1055 kInitializationFailed,
1056 kInitialized
1057 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
1058 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -07001059 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -07001060 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -07001061 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1062 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1063 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1064 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1065 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001066 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001067 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +01001068 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -07001069 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001070 absl::optional<bool>
1071 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
1072 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -07001073 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001074 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
1075 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001076 absl::optional<VideoEncoder::RateControlParameters>
1077 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001078 VideoFrame::UpdateRect last_update_rect_
1079 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001080 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001081 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001082 EncodedImageCallback* encoded_image_callback_
1083 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001084 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001085 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001086 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1087 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001088 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001089 };
1090
mflodmancc3d4422017-08-03 08:27:51 -07001091 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001092 public:
1093 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001094 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001095
perkj26091b12016-09-01 01:17:40 -07001096 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001097 EXPECT_TRUE(
1098 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1099 }
1100
1101 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1102 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001103 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001104 if (!encoded_frame_event_.Wait(timeout_ms))
1105 return false;
perkj26091b12016-09-01 01:17:40 -07001106 {
1107 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001108 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001109 }
1110 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001111 return true;
perkj26091b12016-09-01 01:17:40 -07001112 }
1113
sprangb1ca0732017-02-01 08:38:12 -08001114 void WaitForEncodedFrame(uint32_t expected_width,
1115 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001116 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001117 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001118 }
1119
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001120 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001121 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001122 uint32_t width = 0;
1123 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001124 {
1125 rtc::CritScope lock(&crit_);
1126 width = last_width_;
1127 height = last_height_;
1128 }
1129 EXPECT_EQ(expected_height, height);
1130 EXPECT_EQ(expected_width, width);
1131 }
1132
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001133 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1134 int width = 0;
1135 int height = 0;
1136 {
1137 rtc::CritScope lock(&crit_);
1138 width = last_width_;
1139 height = last_height_;
1140 }
1141 EXPECT_EQ(width % resolution_alignment, 0);
1142 EXPECT_EQ(height % resolution_alignment, 0);
1143 }
1144
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001145 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1146 VideoRotation rotation;
1147 {
1148 rtc::CritScope lock(&crit_);
1149 rotation = last_rotation_;
1150 }
1151 EXPECT_EQ(expected_rotation, rotation);
1152 }
1153
kthelgason2fc52542017-03-03 00:24:41 -08001154 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001155
sprangc5d62e22017-04-02 23:53:04 -07001156 bool WaitForFrame(int64_t timeout_ms) {
1157 return encoded_frame_event_.Wait(timeout_ms);
1158 }
1159
perkj26091b12016-09-01 01:17:40 -07001160 void SetExpectNoFrames() {
1161 rtc::CritScope lock(&crit_);
1162 expect_frames_ = false;
1163 }
1164
asaperssonfab67072017-04-04 05:51:49 -07001165 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001166 rtc::CritScope lock(&crit_);
1167 return number_of_reconfigurations_;
1168 }
1169
asaperssonfab67072017-04-04 05:51:49 -07001170 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001171 rtc::CritScope lock(&crit_);
1172 return min_transmit_bitrate_bps_;
1173 }
1174
Erik Språngd7329ca2019-02-21 21:19:53 +01001175 void SetNumExpectedLayers(size_t num_layers) {
1176 rtc::CritScope lock(&crit_);
1177 num_expected_layers_ = num_layers;
1178 }
1179
Erik Språngb7cb7b52019-02-26 15:52:33 +01001180 int64_t GetLastCaptureTimeMs() const {
1181 rtc::CritScope lock(&crit_);
1182 return last_capture_time_ms_;
1183 }
1184
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001185 std::vector<uint8_t> GetLastEncodedImageData() {
1186 rtc::CritScope lock(&crit_);
1187 return std::move(last_encoded_image_data_);
1188 }
1189
1190 RTPFragmentationHeader GetLastFragmentation() {
1191 rtc::CritScope lock(&crit_);
1192 return std::move(last_fragmentation_);
1193 }
1194
perkj26091b12016-09-01 01:17:40 -07001195 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001196 Result OnEncodedImage(
1197 const EncodedImage& encoded_image,
1198 const CodecSpecificInfo* codec_specific_info,
1199 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001200 rtc::CritScope lock(&crit_);
1201 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001202 last_encoded_image_data_ = std::vector<uint8_t>(
1203 encoded_image.data(), encoded_image.data() + encoded_image.size());
1204 if (fragmentation) {
1205 last_fragmentation_.CopyFrom(*fragmentation);
1206 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001207 uint32_t timestamp = encoded_image.Timestamp();
1208 if (last_timestamp_ != timestamp) {
1209 num_received_layers_ = 1;
1210 } else {
1211 ++num_received_layers_;
1212 }
1213 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001214 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001215 last_width_ = encoded_image._encodedWidth;
1216 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001217 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001218 if (num_received_layers_ == num_expected_layers_) {
1219 encoded_frame_event_.Set();
1220 }
sprangb1ca0732017-02-01 08:38:12 -08001221 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001222 }
1223
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001224 void OnEncoderConfigurationChanged(
1225 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001226 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001227 VideoEncoderConfig::ContentType content_type,
1228 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001229 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001230 ++number_of_reconfigurations_;
1231 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1232 }
1233
perkj26091b12016-09-01 01:17:40 -07001234 rtc::CriticalSection crit_;
1235 TestEncoder* test_encoder_;
1236 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001237 std::vector<uint8_t> last_encoded_image_data_;
1238 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001239 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001240 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001241 uint32_t last_height_ = 0;
1242 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001243 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001244 size_t num_expected_layers_ = 1;
1245 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001246 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001247 int number_of_reconfigurations_ = 0;
1248 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001249 };
1250
Sergey Silkin5ee69672019-07-02 14:18:34 +02001251 class VideoBitrateAllocatorProxyFactory
1252 : public VideoBitrateAllocatorFactory {
1253 public:
1254 VideoBitrateAllocatorProxyFactory()
1255 : bitrate_allocator_factory_(
1256 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1257
1258 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1259 const VideoCodec& codec) override {
1260 rtc::CritScope lock(&crit_);
1261 codec_config_ = codec;
1262 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1263 }
1264
1265 VideoCodec codec_config() const {
1266 rtc::CritScope lock(&crit_);
1267 return codec_config_;
1268 }
1269
1270 private:
1271 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1272
1273 rtc::CriticalSection crit_;
1274 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1275 };
1276
perkj26091b12016-09-01 01:17:40 -07001277 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001278 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001279 int codec_width_;
1280 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001281 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001282 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001283 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001284 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001285 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001286 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001287 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001288 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001289 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001290 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001291};
1292
mflodmancc3d4422017-08-03 08:27:51 -07001293TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001294 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001295 DataRate::BitsPerSec(kTargetBitrateBps),
1296 DataRate::BitsPerSec(kTargetBitrateBps),
1297 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001298 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001299 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001300 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001301 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001302 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001303}
1304
mflodmancc3d4422017-08-03 08:27:51 -07001305TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001306 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001307 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001308 // The encoder will cache up to one frame for a short duration. Adding two
1309 // frames means that the first frame will be dropped and the second frame will
1310 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001311 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001312 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001313 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001314
Henrik Boström381d1092020-05-12 18:49:07 +02001315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001316 DataRate::BitsPerSec(kTargetBitrateBps),
1317 DataRate::BitsPerSec(kTargetBitrateBps),
1318 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001319
Sebastian Janssona3177052018-04-10 13:05:49 +02001320 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001321 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001322 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1323
1324 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001325 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001326}
1327
mflodmancc3d4422017-08-03 08:27:51 -07001328TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001330 DataRate::BitsPerSec(kTargetBitrateBps),
1331 DataRate::BitsPerSec(kTargetBitrateBps),
1332 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001333 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001334 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001335
Henrik Boström381d1092020-05-12 18:49:07 +02001336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1337 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1338 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001339 // The encoder will cache up to one frame for a short duration. Adding two
1340 // frames means that the first frame will be dropped and the second frame will
1341 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001342 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001343 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001344
Henrik Boström381d1092020-05-12 18:49:07 +02001345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001346 DataRate::BitsPerSec(kTargetBitrateBps),
1347 DataRate::BitsPerSec(kTargetBitrateBps),
1348 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001349 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001350 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1351 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001352 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001353}
1354
mflodmancc3d4422017-08-03 08:27:51 -07001355TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001357 DataRate::BitsPerSec(kTargetBitrateBps),
1358 DataRate::BitsPerSec(kTargetBitrateBps),
1359 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001360 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001361 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001362
1363 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001364 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001365
perkja49cbd32016-09-16 07:53:41 -07001366 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001367 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001368 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001369}
1370
mflodmancc3d4422017-08-03 08:27:51 -07001371TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001373 DataRate::BitsPerSec(kTargetBitrateBps),
1374 DataRate::BitsPerSec(kTargetBitrateBps),
1375 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001376
perkja49cbd32016-09-16 07:53:41 -07001377 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001378 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001379
mflodmancc3d4422017-08-03 08:27:51 -07001380 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001381 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001382 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001383 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1384 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001385}
1386
mflodmancc3d4422017-08-03 08:27:51 -07001387TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001389 DataRate::BitsPerSec(kTargetBitrateBps),
1390 DataRate::BitsPerSec(kTargetBitrateBps),
1391 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001392
1393 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001394 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001395 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001396 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1397 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001398 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1399 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001400 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001401 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001402
mflodmancc3d4422017-08-03 08:27:51 -07001403 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001404}
1405
Noah Richards51db4212019-06-12 06:59:12 -07001406TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001407 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001408 DataRate::BitsPerSec(kTargetBitrateBps),
1409 DataRate::BitsPerSec(kTargetBitrateBps),
1410 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001411
1412 rtc::Event frame_destroyed_event;
1413 video_source_.IncomingCapturedFrame(
1414 CreateFakeNativeFrame(1, &frame_destroyed_event));
1415 ExpectDroppedFrame();
1416 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1417 video_stream_encoder_->Stop();
1418}
1419
1420TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1421 // Use the cropping factory.
1422 video_encoder_config_.video_stream_factory =
1423 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1424 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1425 kMaxPayloadLength);
1426 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1427
1428 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001429 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001430 DataRate::BitsPerSec(kTargetBitrateBps),
1431 DataRate::BitsPerSec(kTargetBitrateBps),
1432 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001433 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1434 WaitForEncodedFrame(1);
1435 // The encoder will have been configured once.
1436 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1437 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1438 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1439
1440 // Now send in a fake frame that needs to be cropped as the width/height
1441 // aren't divisible by 4 (see CreateEncoderStreams above).
1442 rtc::Event frame_destroyed_event;
1443 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1444 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1445 ExpectDroppedFrame();
1446 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1447 video_stream_encoder_->Stop();
1448}
1449
Ying Wang9b881ab2020-02-07 14:29:32 +01001450TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001452 DataRate::BitsPerSec(kTargetBitrateBps),
1453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001455 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1456 WaitForEncodedFrame(1);
1457
Henrik Boström381d1092020-05-12 18:49:07 +02001458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001459 DataRate::BitsPerSec(kTargetBitrateBps),
1460 DataRate::BitsPerSec(kTargetBitrateBps),
1461 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001462 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1463 // frames. Adding two frames means that the first frame will be dropped and
1464 // the second frame will be sent to the encoder.
1465 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1466 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1467 WaitForEncodedFrame(3);
1468 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1469 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1470 WaitForEncodedFrame(5);
1471 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1472 video_stream_encoder_->Stop();
1473}
1474
mflodmancc3d4422017-08-03 08:27:51 -07001475TEST_F(VideoStreamEncoderTest,
1476 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps),
1480 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001481 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001482
1483 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001484 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001485 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001486 // The encoder will have been configured once when the first frame is
1487 // received.
1488 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001489
1490 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001491 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001492 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001493 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001494 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001495
1496 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001497 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001498 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001499 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001500 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001501
mflodmancc3d4422017-08-03 08:27:51 -07001502 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001503}
1504
mflodmancc3d4422017-08-03 08:27:51 -07001505TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001507 DataRate::BitsPerSec(kTargetBitrateBps),
1508 DataRate::BitsPerSec(kTargetBitrateBps),
1509 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001510
1511 // Capture a frame and wait for it to synchronize with the encoder thread.
1512 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001513 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001514 // The encoder will have been configured once.
1515 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001516 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1517 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1518
1519 codec_width_ *= 2;
1520 codec_height_ *= 2;
1521 // Capture a frame with a higher resolution and wait for it to synchronize
1522 // with the encoder thread.
1523 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001524 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001525 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1526 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001527 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001528
mflodmancc3d4422017-08-03 08:27:51 -07001529 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001530}
1531
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001532TEST_F(VideoStreamEncoderTest,
1533 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001538
1539 // Capture a frame and wait for it to synchronize with the encoder thread.
1540 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1541 WaitForEncodedFrame(1);
1542
1543 VideoEncoderConfig video_encoder_config;
1544 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1545 // Changing the max payload data length recreates encoder.
1546 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1547 kMaxPayloadLength / 2);
1548
1549 // Capture a frame and wait for it to synchronize with the encoder thread.
1550 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1551 WaitForEncodedFrame(2);
1552 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1553
1554 video_stream_encoder_->Stop();
1555}
1556
Sergey Silkin5ee69672019-07-02 14:18:34 +02001557TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001559 DataRate::BitsPerSec(kTargetBitrateBps),
1560 DataRate::BitsPerSec(kTargetBitrateBps),
1561 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001562
1563 VideoEncoderConfig video_encoder_config;
1564 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1565 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1566 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1567 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1568 kMaxPayloadLength);
1569
1570 // Capture a frame and wait for it to synchronize with the encoder thread.
1571 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1572 WaitForEncodedFrame(1);
1573 // The encoder will have been configured once when the first frame is
1574 // received.
1575 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1576 EXPECT_EQ(kTargetBitrateBps,
1577 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1578 EXPECT_EQ(kStartBitrateBps,
1579 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1580
Sergey Silkin6456e352019-07-08 17:56:40 +02001581 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1582 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001583 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1584 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1585 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1586 kMaxPayloadLength);
1587
1588 // Capture a frame and wait for it to synchronize with the encoder thread.
1589 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1590 WaitForEncodedFrame(2);
1591 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1592 // Bitrate limits have changed - rate allocator should be reconfigured,
1593 // encoder should not be reconfigured.
1594 EXPECT_EQ(kTargetBitrateBps * 2,
1595 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1596 EXPECT_EQ(kStartBitrateBps * 2,
1597 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1598 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1599
1600 video_stream_encoder_->Stop();
1601}
1602
Sergey Silkin6456e352019-07-08 17:56:40 +02001603TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001604 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001606 DataRate::BitsPerSec(kTargetBitrateBps),
1607 DataRate::BitsPerSec(kTargetBitrateBps),
1608 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001609
Sergey Silkincd02eba2020-01-20 14:48:40 +01001610 const uint32_t kMinEncBitrateKbps = 100;
1611 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001612 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001613 /*frame_size_pixels=*/codec_width_ * codec_height_,
1614 /*min_start_bitrate_bps=*/0,
1615 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1616 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001617 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1618
Sergey Silkincd02eba2020-01-20 14:48:40 +01001619 VideoEncoderConfig video_encoder_config;
1620 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1621 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1622 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1623 (kMinEncBitrateKbps + 1) * 1000;
1624 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1625 kMaxPayloadLength);
1626
1627 // When both encoder and app provide bitrate limits, the intersection of
1628 // provided sets should be used.
1629 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1630 WaitForEncodedFrame(1);
1631 EXPECT_EQ(kMaxEncBitrateKbps,
1632 bitrate_allocator_factory_.codec_config().maxBitrate);
1633 EXPECT_EQ(kMinEncBitrateKbps + 1,
1634 bitrate_allocator_factory_.codec_config().minBitrate);
1635
1636 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1637 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1638 (kMinEncBitrateKbps - 1) * 1000;
1639 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1640 kMaxPayloadLength);
1641 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001642 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001643 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001644 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001645 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001646 bitrate_allocator_factory_.codec_config().minBitrate);
1647
Sergey Silkincd02eba2020-01-20 14:48:40 +01001648 video_stream_encoder_->Stop();
1649}
1650
1651TEST_F(VideoStreamEncoderTest,
1652 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001654 DataRate::BitsPerSec(kTargetBitrateBps),
1655 DataRate::BitsPerSec(kTargetBitrateBps),
1656 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001657
1658 const uint32_t kMinAppBitrateKbps = 100;
1659 const uint32_t kMaxAppBitrateKbps = 200;
1660 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1661 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1662 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1663 /*frame_size_pixels=*/codec_width_ * codec_height_,
1664 /*min_start_bitrate_bps=*/0,
1665 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1666 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1667 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1668
1669 VideoEncoderConfig video_encoder_config;
1670 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1671 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1672 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1673 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001674 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1675 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001676
Sergey Silkincd02eba2020-01-20 14:48:40 +01001677 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1678 WaitForEncodedFrame(1);
1679 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001680 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001681 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001682 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001683
1684 video_stream_encoder_->Stop();
1685}
1686
1687TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001688 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001690 DataRate::BitsPerSec(kTargetBitrateBps),
1691 DataRate::BitsPerSec(kTargetBitrateBps),
1692 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001693
1694 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001695 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001696 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001697 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001698 fake_encoder_.SetResolutionBitrateLimits(
1699 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1700
1701 VideoEncoderConfig video_encoder_config;
1702 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1703 video_encoder_config.max_bitrate_bps = 0;
1704 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1705 kMaxPayloadLength);
1706
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001707 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001708 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1709 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001710 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1711 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001712 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1713 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1714
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001715 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001716 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1717 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001718 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1719 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001720 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1721 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1722
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001723 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001724 // encoder for 360p should be used.
1725 video_source_.IncomingCapturedFrame(
1726 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1727 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001728 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1729 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001730 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1731 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1732
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001733 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001734 // ignored.
1735 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1736 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001737 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1738 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001739 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1740 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001741 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1742 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001743 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1744 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1745
1746 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1747 // for 270p should be used.
1748 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1749 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001750 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1751 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001752 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1753 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1754
1755 video_stream_encoder_->Stop();
1756}
1757
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001758TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001759 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001760 DataRate::BitsPerSec(kTargetBitrateBps),
1761 DataRate::BitsPerSec(kTargetBitrateBps),
1762 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001763
1764 VideoEncoderConfig video_encoder_config;
1765 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1766 video_encoder_config.max_bitrate_bps = 0;
1767 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1768 kMaxPayloadLength);
1769
1770 // Encode 720p frame to get the default encoder target bitrate.
1771 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1772 WaitForEncodedFrame(1);
1773 const uint32_t kDefaultTargetBitrateFor720pKbps =
1774 bitrate_allocator_factory_.codec_config()
1775 .simulcastStream[0]
1776 .targetBitrate;
1777
1778 // Set the max recommended encoder bitrate to something lower than the default
1779 // target bitrate.
1780 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1781 1280 * 720, 10 * 1000, 10 * 1000,
1782 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1783 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1784
1785 // Change resolution to trigger encoder reinitialization.
1786 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1787 WaitForEncodedFrame(2);
1788 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1789 WaitForEncodedFrame(3);
1790
1791 // Ensure the target bitrate is capped by the max bitrate.
1792 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1793 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1794 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1795 .simulcastStream[0]
1796 .targetBitrate *
1797 1000,
1798 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1799
1800 video_stream_encoder_->Stop();
1801}
1802
mflodmancc3d4422017-08-03 08:27:51 -07001803TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001804 EXPECT_TRUE(video_source_.has_sinks());
1805 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001806 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001807 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001808 EXPECT_FALSE(video_source_.has_sinks());
1809 EXPECT_TRUE(new_video_source.has_sinks());
1810
mflodmancc3d4422017-08-03 08:27:51 -07001811 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001812}
1813
mflodmancc3d4422017-08-03 08:27:51 -07001814TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001815 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001816 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001817 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001818 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001819}
1820
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001821TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1822 constexpr int kRequestedResolutionAlignment = 7;
1823 video_source_.set_adaptation_enabled(true);
1824 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001826 DataRate::BitsPerSec(kTargetBitrateBps),
1827 DataRate::BitsPerSec(kTargetBitrateBps),
1828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001829
1830 // On the 1st frame, we should have initialized the encoder and
1831 // asked for its resolution requirements.
1832 video_source_.IncomingCapturedFrame(
1833 CreateFrame(1, codec_width_, codec_height_));
1834 WaitForEncodedFrame(1);
1835 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1836 kRequestedResolutionAlignment);
1837
1838 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1839 // (It's up the to the encoder to potentially drop the previous frame,
1840 // to avoid coding back-to-back keyframes.)
1841 video_source_.IncomingCapturedFrame(
1842 CreateFrame(2, codec_width_, codec_height_));
1843 WaitForEncodedFrame(2);
1844 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1845
1846 video_stream_encoder_->Stop();
1847}
1848
Jonathan Yubc771b72017-12-08 17:04:29 -08001849TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1850 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001851 const int kWidth = 1280;
1852 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001853
1854 // We rely on the automatic resolution adaptation, but we handle framerate
1855 // adaptation manually by mocking the stats proxy.
1856 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001857
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001858 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001860 DataRate::BitsPerSec(kTargetBitrateBps),
1861 DataRate::BitsPerSec(kTargetBitrateBps),
1862 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001863 video_stream_encoder_->SetSource(&video_source_,
1864 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001865 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001866 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001868 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1869
Jonathan Yubc771b72017-12-08 17:04:29 -08001870 // Adapt down as far as possible.
1871 rtc::VideoSinkWants last_wants;
1872 int64_t t = 1;
1873 int loop_count = 0;
1874 do {
1875 ++loop_count;
1876 last_wants = video_source_.sink_wants();
1877
1878 // Simulate the framerate we've been asked to adapt to.
1879 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1880 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1881 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1882 mock_stats.input_frame_rate = fps;
1883 stats_proxy_->SetMockStats(mock_stats);
1884
1885 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1886 sink_.WaitForEncodedFrame(t);
1887 t += frame_interval_ms;
1888
mflodmancc3d4422017-08-03 08:27:51 -07001889 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001890 VerifyBalancedModeFpsRange(
1891 video_source_.sink_wants(),
1892 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1893 } while (video_source_.sink_wants().max_pixel_count <
1894 last_wants.max_pixel_count ||
1895 video_source_.sink_wants().max_framerate_fps <
1896 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001897
Jonathan Yubc771b72017-12-08 17:04:29 -08001898 // Verify that we've adapted all the way down.
1899 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001900 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001901 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1902 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001903 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001904 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1905 *video_source_.last_sent_height());
1906 EXPECT_EQ(kMinBalancedFramerateFps,
1907 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001908
Jonathan Yubc771b72017-12-08 17:04:29 -08001909 // Adapt back up the same number of times we adapted down.
1910 for (int i = 0; i < loop_count - 1; ++i) {
1911 last_wants = video_source_.sink_wants();
1912
1913 // Simulate the framerate we've been asked to adapt to.
1914 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1915 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1916 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1917 mock_stats.input_frame_rate = fps;
1918 stats_proxy_->SetMockStats(mock_stats);
1919
1920 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1921 sink_.WaitForEncodedFrame(t);
1922 t += frame_interval_ms;
1923
Henrik Boström91aa7322020-04-28 12:24:33 +02001924 video_stream_encoder_->TriggerCpuUnderuse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001925 VerifyBalancedModeFpsRange(
1926 video_source_.sink_wants(),
1927 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1928 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1929 last_wants.max_pixel_count ||
1930 video_source_.sink_wants().max_framerate_fps >
1931 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001932 }
1933
Åsa Persson8c1bf952018-09-13 10:42:19 +02001934 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001935 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001936 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001937 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1938 EXPECT_EQ((loop_count - 1) * 2,
1939 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001940
mflodmancc3d4422017-08-03 08:27:51 -07001941 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001942}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001943
mflodmancc3d4422017-08-03 08:27:51 -07001944TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02001945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001946 DataRate::BitsPerSec(kTargetBitrateBps),
1947 DataRate::BitsPerSec(kTargetBitrateBps),
1948 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001949 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001950
sprangc5d62e22017-04-02 23:53:04 -07001951 const int kFrameWidth = 1280;
1952 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001953
Åsa Persson8c1bf952018-09-13 10:42:19 +02001954 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001955
kthelgason5e13d412016-12-01 03:59:51 -08001956 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001957 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001958 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001959 frame_timestamp += kFrameIntervalMs;
1960
perkj803d97f2016-11-01 11:45:46 -07001961 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001962 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001963 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001964 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001965 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001966 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001967
asapersson0944a802017-04-07 00:57:58 -07001968 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001969 // wanted resolution.
1970 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1971 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1972 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001973 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001974
1975 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001976 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02001977 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001978 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001979 // Give the encoder queue time to process the change in degradation preference
1980 // by waiting for an encoded frame.
1981 new_video_source.IncomingCapturedFrame(
1982 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1983 sink_.WaitForEncodedFrame(frame_timestamp);
1984 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001985 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001986 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001987
sprangc5d62e22017-04-02 23:53:04 -07001988 // Force an input frame rate to be available, or the adaptation call won't
1989 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001990 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001991 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001992 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001993 stats_proxy_->SetMockStats(stats);
1994
mflodmancc3d4422017-08-03 08:27:51 -07001995 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001996 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001997 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001998 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001999 frame_timestamp += kFrameIntervalMs;
2000
2001 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002002 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002003 EXPECT_EQ(std::numeric_limits<int>::max(),
2004 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002005 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002006
asapersson02465b82017-04-10 01:12:52 -07002007 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002008 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2009 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002010 // Give the encoder queue time to process the change in degradation preference
2011 // by waiting for an encoded frame.
2012 new_video_source.IncomingCapturedFrame(
2013 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2014 sink_.WaitForEncodedFrame(frame_timestamp);
2015 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002016 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07002017
mflodmancc3d4422017-08-03 08:27:51 -07002018 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002019 new_video_source.IncomingCapturedFrame(
2020 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002021 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002022 frame_timestamp += kFrameIntervalMs;
2023
2024 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02002025 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002026
2027 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002028 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002029 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002030 // Give the encoder queue time to process the change in degradation preference
2031 // by waiting for an encoded frame.
2032 new_video_source.IncomingCapturedFrame(
2033 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2034 sink_.WaitForEncodedFrame(frame_timestamp);
2035 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002036 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2037 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002038 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002039 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002040
2041 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002042 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002043 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002044 // Give the encoder queue time to process the change in degradation preference
2045 // by waiting for an encoded frame.
2046 new_video_source.IncomingCapturedFrame(
2047 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2048 sink_.WaitForEncodedFrame(frame_timestamp);
2049 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002050 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2051 EXPECT_EQ(std::numeric_limits<int>::max(),
2052 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002053 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002054
mflodmancc3d4422017-08-03 08:27:51 -07002055 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002056}
2057
mflodmancc3d4422017-08-03 08:27:51 -07002058TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002059 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002060 DataRate::BitsPerSec(kTargetBitrateBps),
2061 DataRate::BitsPerSec(kTargetBitrateBps),
2062 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002063
asaperssonfab67072017-04-04 05:51:49 -07002064 const int kWidth = 1280;
2065 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002066 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002067 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002068 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2069 EXPECT_FALSE(stats.bw_limited_resolution);
2070 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2071
2072 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002073 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002074 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002075 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002076
2077 stats = stats_proxy_->GetStats();
2078 EXPECT_TRUE(stats.bw_limited_resolution);
2079 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2080
2081 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002082 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002083 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002084 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002085
2086 stats = stats_proxy_->GetStats();
2087 EXPECT_FALSE(stats.bw_limited_resolution);
2088 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2089 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2090
mflodmancc3d4422017-08-03 08:27:51 -07002091 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002092}
2093
mflodmancc3d4422017-08-03 08:27:51 -07002094TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002095 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002096 DataRate::BitsPerSec(kTargetBitrateBps),
2097 DataRate::BitsPerSec(kTargetBitrateBps),
2098 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002099
2100 const int kWidth = 1280;
2101 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002102 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002103 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002104 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2105 EXPECT_FALSE(stats.cpu_limited_resolution);
2106 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2107
2108 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002109 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002110 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002111 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002112
2113 stats = stats_proxy_->GetStats();
2114 EXPECT_TRUE(stats.cpu_limited_resolution);
2115 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2116
2117 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002118 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002119 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002120 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002121
2122 stats = stats_proxy_->GetStats();
2123 EXPECT_FALSE(stats.cpu_limited_resolution);
2124 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002125 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002126
mflodmancc3d4422017-08-03 08:27:51 -07002127 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002128}
2129
mflodmancc3d4422017-08-03 08:27:51 -07002130TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002132 DataRate::BitsPerSec(kTargetBitrateBps),
2133 DataRate::BitsPerSec(kTargetBitrateBps),
2134 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002135
asaperssonfab67072017-04-04 05:51:49 -07002136 const int kWidth = 1280;
2137 const int kHeight = 720;
2138 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002139 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002140 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002141 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002142 EXPECT_FALSE(stats.cpu_limited_resolution);
2143 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2144
asaperssonfab67072017-04-04 05:51:49 -07002145 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002146 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002147 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002148 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002149 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002150 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002151 EXPECT_TRUE(stats.cpu_limited_resolution);
2152 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2153
2154 // Set new source with adaptation still enabled.
2155 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002156 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002157 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002158
asaperssonfab67072017-04-04 05:51:49 -07002159 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002160 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002161 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002162 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002163 EXPECT_TRUE(stats.cpu_limited_resolution);
2164 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2165
2166 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002167 video_stream_encoder_->SetSource(&new_video_source,
2168 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002169
asaperssonfab67072017-04-04 05:51:49 -07002170 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002171 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002172 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002173 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002174 EXPECT_FALSE(stats.cpu_limited_resolution);
2175 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2176
2177 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002178 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002179 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002180
asaperssonfab67072017-04-04 05:51:49 -07002181 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002182 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002183 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002184 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002185 EXPECT_TRUE(stats.cpu_limited_resolution);
2186 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2187
asaperssonfab67072017-04-04 05:51:49 -07002188 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002189 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002190 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002191 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002192 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002193 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002194 EXPECT_FALSE(stats.cpu_limited_resolution);
2195 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002196 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002197
mflodmancc3d4422017-08-03 08:27:51 -07002198 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002199}
2200
mflodmancc3d4422017-08-03 08:27:51 -07002201TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002203 DataRate::BitsPerSec(kTargetBitrateBps),
2204 DataRate::BitsPerSec(kTargetBitrateBps),
2205 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002206
asaperssonfab67072017-04-04 05:51:49 -07002207 const int kWidth = 1280;
2208 const int kHeight = 720;
2209 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002210 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002211 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002212 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002213 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002214 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002215
2216 // Set new source with adaptation still enabled.
2217 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002218 video_stream_encoder_->SetSource(&new_video_source,
2219 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002220
asaperssonfab67072017-04-04 05:51:49 -07002221 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002222 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002223 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002224 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002225 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002226 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002227
asaperssonfab67072017-04-04 05:51:49 -07002228 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002229 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002230 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002231 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002232 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002233 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002234 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002235 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002236
asaperssonfab67072017-04-04 05:51:49 -07002237 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002238 video_stream_encoder_->SetSource(&new_video_source,
2239 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002240
asaperssonfab67072017-04-04 05:51:49 -07002241 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002242 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002243 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002244 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002245 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002246 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002247
asapersson02465b82017-04-10 01:12:52 -07002248 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002249 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002250 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002251
asaperssonfab67072017-04-04 05:51:49 -07002252 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002253 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002254 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002255 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002256 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002257 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2258 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002259
mflodmancc3d4422017-08-03 08:27:51 -07002260 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002261}
2262
mflodmancc3d4422017-08-03 08:27:51 -07002263TEST_F(VideoStreamEncoderTest,
2264 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002265 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002266 DataRate::BitsPerSec(kTargetBitrateBps),
2267 DataRate::BitsPerSec(kTargetBitrateBps),
2268 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002269
2270 const int kWidth = 1280;
2271 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002272 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002273 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002274 video_source_.IncomingCapturedFrame(
2275 CreateFrame(timestamp_ms, kWidth, kHeight));
2276 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002277 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2278 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2279 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2280
2281 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002282 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002283 timestamp_ms += kFrameIntervalMs;
2284 video_source_.IncomingCapturedFrame(
2285 CreateFrame(timestamp_ms, kWidth, kHeight));
2286 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002287 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2288 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2289 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2290
2291 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002292 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002293 timestamp_ms += kFrameIntervalMs;
2294 video_source_.IncomingCapturedFrame(
2295 CreateFrame(timestamp_ms, kWidth, kHeight));
2296 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002297 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2298 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2299 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2300
Niels Möller4db138e2018-04-19 09:04:13 +02002301 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002302 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002303
2304 VideoEncoderConfig video_encoder_config;
2305 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2306 // Make format different, to force recreation of encoder.
2307 video_encoder_config.video_format.parameters["foo"] = "foo";
2308 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002309 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002310 timestamp_ms += kFrameIntervalMs;
2311 video_source_.IncomingCapturedFrame(
2312 CreateFrame(timestamp_ms, kWidth, kHeight));
2313 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002314 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2315 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2316 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2317
mflodmancc3d4422017-08-03 08:27:51 -07002318 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002319}
2320
mflodmancc3d4422017-08-03 08:27:51 -07002321TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002322 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002323 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002324 DataRate::BitsPerSec(kTargetBitrateBps),
2325 DataRate::BitsPerSec(kTargetBitrateBps),
2326 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2327
2328 const int kWidth = 1280;
2329 const int kHeight = 720;
2330 int sequence = 1;
2331
2332 // Enable BALANCED preference, no initial limitation.
2333 test::FrameForwarder source;
2334 video_stream_encoder_->SetSource(&source,
2335 webrtc::DegradationPreference::BALANCED);
2336 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2337 WaitForEncodedFrame(sequence++);
2338 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2339 EXPECT_FALSE(stats.cpu_limited_resolution);
2340 EXPECT_FALSE(stats.cpu_limited_framerate);
2341 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2342
2343 // Trigger CPU overuse, should now adapt down.
2344 video_stream_encoder_->TriggerCpuOveruse();
2345 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2346 WaitForEncodedFrame(sequence++);
2347 stats = stats_proxy_->GetStats();
2348 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2349
2350 // Set new degradation preference should clear restrictions since we changed
2351 // from BALANCED.
2352 video_stream_encoder_->SetSource(
2353 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2354 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2355 WaitForEncodedFrame(sequence++);
2356 stats = stats_proxy_->GetStats();
2357 EXPECT_FALSE(stats.cpu_limited_resolution);
2358 EXPECT_FALSE(stats.cpu_limited_framerate);
2359 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2360
2361 // Force an input frame rate to be available, or the adaptation call won't
2362 // know what framerate to adapt from.
2363 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2364 mock_stats.input_frame_rate = 30;
2365 stats_proxy_->SetMockStats(mock_stats);
2366 video_stream_encoder_->TriggerCpuOveruse();
2367 stats_proxy_->ResetMockStats();
2368 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2369 WaitForEncodedFrame(sequence++);
2370
2371 // We have now adapted once.
2372 stats = stats_proxy_->GetStats();
2373 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2374
2375 // Back to BALANCED, should clear the restrictions again.
2376 video_stream_encoder_->SetSource(&source,
2377 webrtc::DegradationPreference::BALANCED);
2378 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2379 WaitForEncodedFrame(sequence++);
2380 stats = stats_proxy_->GetStats();
2381 EXPECT_FALSE(stats.cpu_limited_resolution);
2382 EXPECT_FALSE(stats.cpu_limited_framerate);
2383 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2384
2385 video_stream_encoder_->Stop();
2386}
2387
2388TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002389 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002391 DataRate::BitsPerSec(kTargetBitrateBps),
2392 DataRate::BitsPerSec(kTargetBitrateBps),
2393 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002394
asapersson0944a802017-04-07 00:57:58 -07002395 const int kWidth = 1280;
2396 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002397 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002398
asaperssonfab67072017-04-04 05:51:49 -07002399 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002400 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002401 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002402 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002403 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002404 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2405
asapersson02465b82017-04-10 01:12:52 -07002406 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002407 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002408 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002409 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002410 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002411 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002412 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002413 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2414
2415 // Set new source with adaptation still enabled.
2416 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002417 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002418 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002419
2420 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002421 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002422 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002423 stats = stats_proxy_->GetStats();
2424 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002425 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002426 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2427
sprangc5d62e22017-04-02 23:53:04 -07002428 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002429 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002430 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002431 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002432 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002433 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002434 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002435 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002436 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002437 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002438 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2439
sprangc5d62e22017-04-02 23:53:04 -07002440 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002441 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002442 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2443 mock_stats.input_frame_rate = 30;
2444 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002445 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002446 stats_proxy_->ResetMockStats();
2447
2448 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002449 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002450 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002451
2452 // Framerate now adapted.
2453 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002454 EXPECT_FALSE(stats.cpu_limited_resolution);
2455 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002456 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2457
2458 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002459 video_stream_encoder_->SetSource(&new_video_source,
2460 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002461 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002462 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002463 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002464
2465 stats = stats_proxy_->GetStats();
2466 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002467 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002468 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2469
2470 // Try to trigger overuse. Should not succeed.
2471 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002472 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002473 stats_proxy_->ResetMockStats();
2474
2475 stats = stats_proxy_->GetStats();
2476 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002477 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002478 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2479
2480 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002481 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002482 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002483 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002484 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002485 stats = stats_proxy_->GetStats();
2486 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002487 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002488 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002489
2490 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002491 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002492 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002493 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002494 stats = stats_proxy_->GetStats();
2495 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002496 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002497 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2498
2499 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002500 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002501 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002502 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002503 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002504 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002505 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002506 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002507 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002508 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002509 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2510
2511 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002512 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002513 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002514 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002515 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002516 stats = stats_proxy_->GetStats();
2517 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002518 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002519 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002520 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002521
mflodmancc3d4422017-08-03 08:27:51 -07002522 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002523}
2524
mflodmancc3d4422017-08-03 08:27:51 -07002525TEST_F(VideoStreamEncoderTest,
2526 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002527 const int kWidth = 1280;
2528 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002530 DataRate::BitsPerSec(kTargetBitrateBps),
2531 DataRate::BitsPerSec(kTargetBitrateBps),
2532 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002533
asaperssonfab67072017-04-04 05:51:49 -07002534 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002535 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002536
asaperssonfab67072017-04-04 05:51:49 -07002537 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002538 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002539
asaperssonfab67072017-04-04 05:51:49 -07002540 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002541 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002542
asaperssonfab67072017-04-04 05:51:49 -07002543 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002544 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002545
kthelgason876222f2016-11-29 01:44:11 -08002546 // Expect a scale down.
2547 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002548 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002549
asapersson02465b82017-04-10 01:12:52 -07002550 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002551 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002552 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002554
asaperssonfab67072017-04-04 05:51:49 -07002555 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002556 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002557 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002558 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002559
asaperssonfab67072017-04-04 05:51:49 -07002560 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002561 EXPECT_EQ(std::numeric_limits<int>::max(),
2562 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002563
asaperssonfab67072017-04-04 05:51:49 -07002564 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002565 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002566 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002567 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002568
asapersson02465b82017-04-10 01:12:52 -07002569 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002570 EXPECT_EQ(std::numeric_limits<int>::max(),
2571 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002572
mflodmancc3d4422017-08-03 08:27:51 -07002573 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002574}
2575
mflodmancc3d4422017-08-03 08:27:51 -07002576TEST_F(VideoStreamEncoderTest,
2577 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002578 const int kWidth = 1280;
2579 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002580 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002581 DataRate::BitsPerSec(kTargetBitrateBps),
2582 DataRate::BitsPerSec(kTargetBitrateBps),
2583 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002584
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002585 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002586 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002587 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002588 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002589
2590 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002591 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002592 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002593 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2594 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2595
2596 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002597 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002598 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002599 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2600 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2601 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2602
2603 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002604 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002605 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2606 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2607 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2608
mflodmancc3d4422017-08-03 08:27:51 -07002609 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002610}
2611
mflodmancc3d4422017-08-03 08:27:51 -07002612TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002613 const int kWidth = 1280;
2614 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002616 DataRate::BitsPerSec(kTargetBitrateBps),
2617 DataRate::BitsPerSec(kTargetBitrateBps),
2618 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002619
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002620 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002621 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002622 video_stream_encoder_->SetSource(&source,
2623 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002624 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2625 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002626 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002627
2628 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002629 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002630 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2632 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2633 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2634
2635 // Trigger adapt down for same input resolution, expect no change.
2636 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2637 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002638 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002639 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2641 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2642
2643 // Trigger adapt down for larger input resolution, expect no change.
2644 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2645 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002646 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002647 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2649 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2650
mflodmancc3d4422017-08-03 08:27:51 -07002651 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002652}
2653
mflodmancc3d4422017-08-03 08:27:51 -07002654TEST_F(VideoStreamEncoderTest,
2655 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002656 const int kWidth = 1280;
2657 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002659 DataRate::BitsPerSec(kTargetBitrateBps),
2660 DataRate::BitsPerSec(kTargetBitrateBps),
2661 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002662
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002663 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002664 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002665 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002666 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002667
2668 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002669 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002670 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002671 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2672 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2673
2674 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002675 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002676 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2678 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2679
mflodmancc3d4422017-08-03 08:27:51 -07002680 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002681}
2682
mflodmancc3d4422017-08-03 08:27:51 -07002683TEST_F(VideoStreamEncoderTest,
2684 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002685 const int kWidth = 1280;
2686 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002688 DataRate::BitsPerSec(kTargetBitrateBps),
2689 DataRate::BitsPerSec(kTargetBitrateBps),
2690 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002691
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002692 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002693 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002694 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002695 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002696
2697 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002698 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002699 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002700 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002701 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2702
2703 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002704 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002705 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002706 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2708
mflodmancc3d4422017-08-03 08:27:51 -07002709 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002710}
2711
mflodmancc3d4422017-08-03 08:27:51 -07002712TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002713 const int kWidth = 1280;
2714 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002716 DataRate::BitsPerSec(kTargetBitrateBps),
2717 DataRate::BitsPerSec(kTargetBitrateBps),
2718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002719
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002720 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002721 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002722 video_stream_encoder_->SetSource(&source,
2723 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002724
2725 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2726 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002727 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002728 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2731
2732 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002733 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002734 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2737 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2738
mflodmancc3d4422017-08-03 08:27:51 -07002739 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002740}
2741
mflodmancc3d4422017-08-03 08:27:51 -07002742TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002743 const int kWidth = 1280;
2744 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002746 DataRate::BitsPerSec(kTargetBitrateBps),
2747 DataRate::BitsPerSec(kTargetBitrateBps),
2748 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002749
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002750 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002751 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002752 video_stream_encoder_->SetSource(&source,
2753 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002754
2755 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2756 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002757 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002758 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2759 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2760 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2761
2762 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002763 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002764 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002765 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2766 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2767 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2768
mflodmancc3d4422017-08-03 08:27:51 -07002769 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002770}
2771
mflodmancc3d4422017-08-03 08:27:51 -07002772TEST_F(VideoStreamEncoderTest,
2773 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002774 const int kWidth = 1280;
2775 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002777 DataRate::BitsPerSec(kTargetBitrateBps),
2778 DataRate::BitsPerSec(kTargetBitrateBps),
2779 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002780
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002781 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002782 AdaptingFrameForwarder source;
2783 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002784 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002785 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002786
2787 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002788 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002789 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002790 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2791 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2792
2793 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002794 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002795 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002796 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002797 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002798 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2799 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2800
2801 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002802 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002803 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002804 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2805 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2806 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2807
mflodmancc3d4422017-08-03 08:27:51 -07002808 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002809}
2810
mflodmancc3d4422017-08-03 08:27:51 -07002811TEST_F(VideoStreamEncoderTest,
2812 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002813 const int kWidth = 1280;
2814 const int kHeight = 720;
2815 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002817 DataRate::BitsPerSec(kTargetBitrateBps),
2818 DataRate::BitsPerSec(kTargetBitrateBps),
2819 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002820
2821 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2822 stats.input_frame_rate = kInputFps;
2823 stats_proxy_->SetMockStats(stats);
2824
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002825 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002826 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2827 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002828 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002829
2830 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002831 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002832 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2833 sink_.WaitForEncodedFrame(2);
2834 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2835
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002836 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002837 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002838 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002839 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002840 // Give the encoder queue time to process the change in degradation preference
2841 // by waiting for an encoded frame.
2842 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2843 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002844 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002845
2846 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002848 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2849 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002850 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2851
2852 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002854 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002855
mflodmancc3d4422017-08-03 08:27:51 -07002856 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002857}
2858
mflodmancc3d4422017-08-03 08:27:51 -07002859TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002860 const int kWidth = 1280;
2861 const int kHeight = 720;
2862 const size_t kNumFrames = 10;
2863
Henrik Boström381d1092020-05-12 18:49:07 +02002864 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002865 DataRate::BitsPerSec(kTargetBitrateBps),
2866 DataRate::BitsPerSec(kTargetBitrateBps),
2867 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002868
asaperssond0de2952017-04-21 01:47:31 -07002869 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002870 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002871 video_source_.set_adaptation_enabled(true);
2872
2873 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2874 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2875
2876 int downscales = 0;
2877 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002878 video_source_.IncomingCapturedFrame(
2879 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2880 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002881
asaperssonfab67072017-04-04 05:51:49 -07002882 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002883 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002884 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002885 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002886
2887 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2888 ++downscales;
2889
2890 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2891 EXPECT_EQ(downscales,
2892 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2893 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002894 }
mflodmancc3d4422017-08-03 08:27:51 -07002895 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002896}
2897
mflodmancc3d4422017-08-03 08:27:51 -07002898TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002899 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2900 const int kWidth = 1280;
2901 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002902 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002903 DataRate::BitsPerSec(kTargetBitrateBps),
2904 DataRate::BitsPerSec(kTargetBitrateBps),
2905 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002906
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002907 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002908 AdaptingFrameForwarder source;
2909 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002910 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002911 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002912
Åsa Persson8c1bf952018-09-13 10:42:19 +02002913 int64_t timestamp_ms = kFrameIntervalMs;
2914 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002915 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002916 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002917 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2918 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2919
2920 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002921 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002922 timestamp_ms += kFrameIntervalMs;
2923 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2924 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002925 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002926 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2927 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2928
2929 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02002930 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002931 timestamp_ms += kFrameIntervalMs;
2932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002933 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002934 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002935 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2936 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2937
2938 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002939 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002940 timestamp_ms += kFrameIntervalMs;
2941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2942 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002943 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002944 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2945 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2946
2947 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02002948 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002949 timestamp_ms += kFrameIntervalMs;
2950 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002951 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002952 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002953 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2954 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2955
mflodmancc3d4422017-08-03 08:27:51 -07002956 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002957}
2958
mflodmancc3d4422017-08-03 08:27:51 -07002959TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002960 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2961 const int kWidth = 1280;
2962 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002964 DataRate::BitsPerSec(kTargetBitrateBps),
2965 DataRate::BitsPerSec(kTargetBitrateBps),
2966 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002967
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002968 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002969 AdaptingFrameForwarder source;
2970 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002971 video_stream_encoder_->SetSource(&source,
2972 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002973
Åsa Persson8c1bf952018-09-13 10:42:19 +02002974 int64_t timestamp_ms = kFrameIntervalMs;
2975 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002976 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002977 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002978 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2979 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2980
2981 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002982 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002983 timestamp_ms += kFrameIntervalMs;
2984 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2985 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002986 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2987 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2988 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2989
2990 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002991 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002992 timestamp_ms += kFrameIntervalMs;
2993 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002994 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002995 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002996 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2997 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2998
2999 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003000 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003001 timestamp_ms += kFrameIntervalMs;
3002 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3003 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003004 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3005 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3006 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3007
3008 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003009 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003010 timestamp_ms += kFrameIntervalMs;
3011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003012 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003013 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003014 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3015 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3016
mflodmancc3d4422017-08-03 08:27:51 -07003017 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003018}
3019
Sergey Silkin41c650b2019-10-14 13:12:19 +02003020TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3021 fake_encoder_.SetResolutionBitrateLimits(
3022 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3023
Henrik Boström381d1092020-05-12 18:49:07 +02003024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003025 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3026 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3027 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3028 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003029
3030 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3031 AdaptingFrameForwarder source;
3032 source.set_adaptation_enabled(true);
3033 video_stream_encoder_->SetSource(
3034 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3035
3036 // Insert 720p frame.
3037 int64_t timestamp_ms = kFrameIntervalMs;
3038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3039 WaitForEncodedFrame(1280, 720);
3040
3041 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003043 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3044 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3045 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3046 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003047 video_stream_encoder_->TriggerQualityLow();
3048
3049 // Insert 720p frame. It should be downscaled and encoded.
3050 timestamp_ms += kFrameIntervalMs;
3051 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3052 WaitForEncodedFrame(960, 540);
3053
3054 // Trigger adapt up. Higher resolution should not be requested duo to lack
3055 // of bitrate.
3056 video_stream_encoder_->TriggerQualityHigh();
3057 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3058
3059 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003061 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3062 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3063 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3064 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003065
3066 // Trigger adapt up. Higher resolution should be requested.
3067 video_stream_encoder_->TriggerQualityHigh();
3068 VerifyFpsMaxResolutionMax(source.sink_wants());
3069
3070 video_stream_encoder_->Stop();
3071}
3072
3073TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3074 fake_encoder_.SetResolutionBitrateLimits(
3075 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3076
3077 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003079 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3080 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3081 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3082 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003083
3084 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3085 AdaptingFrameForwarder source;
3086 source.set_adaptation_enabled(true);
3087 video_stream_encoder_->SetSource(
3088 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3089
3090 // Insert 720p frame. It should be dropped and lower resolution should be
3091 // requested.
3092 int64_t timestamp_ms = kFrameIntervalMs;
3093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3094 ExpectDroppedFrame();
3095 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3096
3097 // Insert 720p frame. It should be downscaled and encoded.
3098 timestamp_ms += kFrameIntervalMs;
3099 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3100 WaitForEncodedFrame(960, 540);
3101
3102 video_stream_encoder_->Stop();
3103}
3104
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003105class BalancedDegradationTest : public VideoStreamEncoderTest {
3106 protected:
3107 void SetupTest() {
3108 // Reset encoder for field trials to take effect.
3109 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003110 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003111
3112 // Enable BALANCED preference.
3113 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003114 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3115 }
3116
3117 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003119 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3120 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003121 }
3122
Åsa Persson45b176f2019-09-30 11:19:05 +02003123 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003124 timestamp_ms_ += kFrameIntervalMs;
3125 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003126 }
3127
3128 void InsertFrameAndWaitForEncoded() {
3129 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003130 sink_.WaitForEncodedFrame(timestamp_ms_);
3131 }
3132
3133 const int kWidth = 640; // pixels:640x360=230400
3134 const int kHeight = 360;
3135 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3136 int64_t timestamp_ms_ = 0;
3137 AdaptingFrameForwarder source_;
3138};
3139
3140TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3141 test::ScopedFieldTrials field_trials(
3142 "WebRTC-Video-BalancedDegradationSettings/"
3143 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3144 SetupTest();
3145
3146 // Force input frame rate.
3147 const int kInputFps = 24;
3148 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3149 stats.input_frame_rate = kInputFps;
3150 stats_proxy_->SetMockStats(stats);
3151
Åsa Persson45b176f2019-09-30 11:19:05 +02003152 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003153 VerifyFpsMaxResolutionMax(source_.sink_wants());
3154
3155 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003156 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3157 // QP samples.
3158 EXPECT_FALSE(
3159 video_stream_encoder_
3160 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003161 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3162
3163 video_stream_encoder_->Stop();
3164}
3165
3166TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3167 test::ScopedFieldTrials field_trials(
3168 "WebRTC-Video-BalancedDegradationSettings/"
3169 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3170 SetupTest();
3171
3172 // Force input frame rate.
3173 const int kInputFps = 25;
3174 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3175 stats.input_frame_rate = kInputFps;
3176 stats_proxy_->SetMockStats(stats);
3177
Åsa Persson45b176f2019-09-30 11:19:05 +02003178 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003179 VerifyFpsMaxResolutionMax(source_.sink_wants());
3180
3181 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003182 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3183 // samples.
3184 EXPECT_TRUE(
3185 video_stream_encoder_
3186 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003187 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3188
3189 video_stream_encoder_->Stop();
3190}
3191
3192TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3193 test::ScopedFieldTrials field_trials(
3194 "WebRTC-Video-BalancedDegradationSettings/"
3195 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3196 SetupTest();
3197
3198 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3199
Åsa Persson45b176f2019-09-30 11:19:05 +02003200 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003201 VerifyFpsMaxResolutionMax(source_.sink_wants());
3202
3203 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3204 video_stream_encoder_->TriggerQualityLow();
3205 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3206
3207 video_stream_encoder_->Stop();
3208}
3209
Åsa Perssonccfb3402019-09-25 15:13:04 +02003210TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003211 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003212 "WebRTC-Video-BalancedDegradationSettings/"
3213 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003214 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003215
Åsa Persson1b247f12019-08-14 17:26:39 +02003216 const int kMinBitrateBps = 425000;
3217 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003218 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003219
Åsa Persson45b176f2019-09-30 11:19:05 +02003220 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003221 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003222 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3223
3224 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3225 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003226 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003227 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003228 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3229
3230 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3231 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003232 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003233 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003234 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3235
Åsa Persson30ab0152019-08-27 12:22:33 +02003236 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3237 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003238 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003239 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3240 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003241 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3242
3243 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003244 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003245 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003246 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003247
Åsa Persson30ab0152019-08-27 12:22:33 +02003248 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003249 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003250 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003251 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003252 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003253 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3254
3255 video_stream_encoder_->Stop();
3256}
3257
Åsa Perssonccfb3402019-09-25 15:13:04 +02003258TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003259 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3260 test::ScopedFieldTrials field_trials(
3261 "WebRTC-Video-BalancedDegradationSettings/"
3262 "pixels:57600|129600|230400,fps:7|24|24/");
3263 SetupTest();
3264 OnBitrateUpdated(kLowTargetBitrateBps);
3265
3266 VerifyNoLimitation(source_.sink_wants());
3267
3268 // Insert frame, expect scaled down:
3269 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3270 InsertFrame();
3271 EXPECT_FALSE(WaitForFrame(1000));
3272 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3273 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3274
3275 // Insert frame, expect scaled down:
3276 // resolution (320x180@24fps).
3277 InsertFrame();
3278 EXPECT_FALSE(WaitForFrame(1000));
3279 EXPECT_LT(source_.sink_wants().max_pixel_count,
3280 source_.last_wants().max_pixel_count);
3281 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3282
3283 // Frame should not be dropped (min pixels per frame reached).
3284 InsertFrameAndWaitForEncoded();
3285
3286 video_stream_encoder_->Stop();
3287}
3288
3289TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003290 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003291 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003292 "WebRTC-Video-BalancedDegradationSettings/"
3293 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003294 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003295
Åsa Persson30ab0152019-08-27 12:22:33 +02003296 const int kResolutionMinBitrateBps = 435000;
3297 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003298 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003299
Åsa Persson45b176f2019-09-30 11:19:05 +02003300 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003301 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003302 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3303
3304 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3305 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003306 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003307 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003308 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3309
3310 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3311 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003312 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003313 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003314 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3315
3316 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3317 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003318 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003319 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003320 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3321
Åsa Persson30ab0152019-08-27 12:22:33 +02003322 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3323 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003324 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003325 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003326 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3327
3328 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3329 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003330 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003331 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3332
3333 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003334 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003335 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003336 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003337 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003338 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3339
3340 video_stream_encoder_->Stop();
3341}
3342
Åsa Perssonccfb3402019-09-25 15:13:04 +02003343TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003344 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003345 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003346 "WebRTC-Video-BalancedDegradationSettings/"
3347 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003348 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003349
Åsa Persson30ab0152019-08-27 12:22:33 +02003350 const int kMinBitrateBps = 425000;
3351 const int kTooLowMinBitrateBps = 424000;
3352 const int kResolutionMinBitrateBps = 435000;
3353 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003354 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003355
Åsa Persson45b176f2019-09-30 11:19:05 +02003356 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003357 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003358 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3359
3360 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3361 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003362 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003363 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003364 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3365
3366 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3367 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003368 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003369 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003370 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3371
3372 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3373 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003374 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003375 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003376 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3377
3378 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3379 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003380 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003381 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003384 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003385 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003386 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003387 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003388 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3389
3390 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003391 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003392 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003393 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003394 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3395
3396 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003397 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003398 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003399 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003400 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003401 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3402
Åsa Persson1b247f12019-08-14 17:26:39 +02003403 video_stream_encoder_->Stop();
3404}
3405
mflodmancc3d4422017-08-03 08:27:51 -07003406TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003407 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3408 const int kWidth = 1280;
3409 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003411 DataRate::BitsPerSec(kTargetBitrateBps),
3412 DataRate::BitsPerSec(kTargetBitrateBps),
3413 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003414
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003415 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003416 AdaptingFrameForwarder source;
3417 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003418 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003419 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003420
Åsa Persson8c1bf952018-09-13 10:42:19 +02003421 int64_t timestamp_ms = kFrameIntervalMs;
3422 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003423 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003424 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003425 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3427 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3428 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
3430 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003431 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003432 timestamp_ms += kFrameIntervalMs;
3433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3434 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003435 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003436 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3438 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3439 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3440
3441 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003442 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003443 timestamp_ms += kFrameIntervalMs;
3444 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3445 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003446 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003447 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3448 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3449 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3450 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3451
Jonathan Yubc771b72017-12-08 17:04:29 -08003452 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003453 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003454 timestamp_ms += kFrameIntervalMs;
3455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3456 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003457 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003458 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3459 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003460 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003461 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3462
Jonathan Yubc771b72017-12-08 17:04:29 -08003463 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003464 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003465 timestamp_ms += kFrameIntervalMs;
3466 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3467 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003468 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003469 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003470 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3471 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3472 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3473 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3474
Jonathan Yubc771b72017-12-08 17:04:29 -08003475 // Trigger quality adapt down, expect no change (min resolution reached).
3476 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003477 timestamp_ms += kFrameIntervalMs;
3478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3479 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003480 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3481 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3482 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3483 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3484 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3485
3486 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003487 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003488 timestamp_ms += kFrameIntervalMs;
3489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3490 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003491 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003492 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3494 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3495 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3496
3497 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003498 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003499 timestamp_ms += kFrameIntervalMs;
3500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3501 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003502 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3503 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3505 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3506 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3507
3508 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003509 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003510 timestamp_ms += kFrameIntervalMs;
3511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3512 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003513 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003514 last_wants = source.sink_wants();
3515 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003517 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003518 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3519
3520 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003521 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003522 timestamp_ms += kFrameIntervalMs;
3523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3524 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003525 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003526 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003528 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003529 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3530
3531 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003532 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003533 timestamp_ms += kFrameIntervalMs;
3534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003535 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003536 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003537 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003538 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3539 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003540 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003541 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003542
mflodmancc3d4422017-08-03 08:27:51 -07003543 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003544}
3545
mflodmancc3d4422017-08-03 08:27:51 -07003546TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003547 const int kWidth = 640;
3548 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003549
Henrik Boström381d1092020-05-12 18:49:07 +02003550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003551 DataRate::BitsPerSec(kTargetBitrateBps),
3552 DataRate::BitsPerSec(kTargetBitrateBps),
3553 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003554
perkj803d97f2016-11-01 11:45:46 -07003555 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003556 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003557 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003558 }
3559
mflodmancc3d4422017-08-03 08:27:51 -07003560 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003561 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003562 video_source_.IncomingCapturedFrame(CreateFrame(
3563 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003564 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003565 }
3566
mflodmancc3d4422017-08-03 08:27:51 -07003567 video_stream_encoder_->Stop();
3568 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003569 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003570
Ying Wangef3998f2019-12-09 13:06:53 +01003571 EXPECT_METRIC_EQ(
3572 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3573 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003574 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3575}
3576
mflodmancc3d4422017-08-03 08:27:51 -07003577TEST_F(VideoStreamEncoderTest,
3578 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003580 DataRate::BitsPerSec(kTargetBitrateBps),
3581 DataRate::BitsPerSec(kTargetBitrateBps),
3582 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003583 const int kWidth = 640;
3584 const int kHeight = 360;
3585
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003586 video_stream_encoder_->SetSource(&video_source_,
3587 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003588
3589 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3590 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003591 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003592 }
3593
mflodmancc3d4422017-08-03 08:27:51 -07003594 video_stream_encoder_->Stop();
3595 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003596 stats_proxy_.reset();
3597
3598 EXPECT_EQ(0,
3599 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3600}
3601
mflodmancc3d4422017-08-03 08:27:51 -07003602TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003603 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003604 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003605
3606 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003607 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003608 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003609 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3610 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003611
sprang57c2fff2017-01-16 06:24:02 -08003612 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003613 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003614 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003615 DataRate::BitsPerSec(kLowTargetBitrateBps),
3616 DataRate::BitsPerSec(kLowTargetBitrateBps),
3617 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003618
sprang57c2fff2017-01-16 06:24:02 -08003619 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003620 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3621 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003622 VideoBitrateAllocation bitrate_allocation =
3623 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003624 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003625 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003626 // TODO(srte): The use of millisecs here looks like an error, but the tests
3627 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003628 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003629
3630 // Not called on second frame.
3631 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3632 .Times(0);
3633 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003634 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3635 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003636 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003637
3638 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003639 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3640 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003641 const int64_t start_time_ms = rtc::TimeMillis();
3642 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3643 video_source_.IncomingCapturedFrame(
3644 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3645 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003646 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003647 }
3648
3649 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003650 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003651
mflodmancc3d4422017-08-03 08:27:51 -07003652 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003653}
3654
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003655TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3656 // 2 TLs configured, temporal layers supported by encoder.
3657 const int kNumTemporalLayers = 2;
3658 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3659 fake_encoder_.SetTemporalLayersSupported(0, true);
3660
3661 // Bitrate allocated across temporal layers.
3662 const int kTl0Bps = kTargetBitrateBps *
3663 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003664 kNumTemporalLayers, /*temporal_id*/ 0,
3665 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003666 const int kTl1Bps = kTargetBitrateBps *
3667 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003668 kNumTemporalLayers, /*temporal_id*/ 1,
3669 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003670 VideoBitrateAllocation expected_bitrate;
3671 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3672 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3673
3674 VerifyAllocatedBitrate(expected_bitrate);
3675 video_stream_encoder_->Stop();
3676}
3677
3678TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3679 // 2 TLs configured, temporal layers not supported by encoder.
3680 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3681 fake_encoder_.SetTemporalLayersSupported(0, false);
3682
3683 // Temporal layers not supported by the encoder.
3684 // Total bitrate should be at ti:0.
3685 VideoBitrateAllocation expected_bitrate;
3686 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3687
3688 VerifyAllocatedBitrate(expected_bitrate);
3689 video_stream_encoder_->Stop();
3690}
3691
3692TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3693 // 2 TLs configured, temporal layers only supported for first stream.
3694 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3695 fake_encoder_.SetTemporalLayersSupported(0, true);
3696 fake_encoder_.SetTemporalLayersSupported(1, false);
3697
3698 const int kS0Bps = 150000;
3699 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003700 kS0Bps *
3701 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3702 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003703 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003704 kS0Bps *
3705 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3706 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003707 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3708 // Temporal layers not supported by si:1.
3709 VideoBitrateAllocation expected_bitrate;
3710 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3711 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3712 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3713
3714 VerifyAllocatedBitrate(expected_bitrate);
3715 video_stream_encoder_->Stop();
3716}
3717
Niels Möller7dc26b72017-12-06 10:27:48 +01003718TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3719 const int kFrameWidth = 1280;
3720 const int kFrameHeight = 720;
3721 const int kFramerate = 24;
3722
Henrik Boström381d1092020-05-12 18:49:07 +02003723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003724 DataRate::BitsPerSec(kTargetBitrateBps),
3725 DataRate::BitsPerSec(kTargetBitrateBps),
3726 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003727 test::FrameForwarder source;
3728 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003729 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003730
3731 // Insert a single frame, triggering initial configuration.
3732 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3733 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3734
3735 EXPECT_EQ(
3736 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3737 kDefaultFramerate);
3738
3739 // Trigger reconfigure encoder (without resetting the entire instance).
3740 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003741 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003742 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3743 video_encoder_config.number_of_streams = 1;
3744 video_encoder_config.video_stream_factory =
3745 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3746 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003747 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003748 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3749
3750 // Detector should be updated with fps limit from codec config.
3751 EXPECT_EQ(
3752 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3753 kFramerate);
3754
3755 // Trigger overuse, max framerate should be reduced.
3756 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3757 stats.input_frame_rate = kFramerate;
3758 stats_proxy_->SetMockStats(stats);
3759 video_stream_encoder_->TriggerCpuOveruse();
3760 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3761 int adapted_framerate =
3762 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3763 EXPECT_LT(adapted_framerate, kFramerate);
3764
3765 // Trigger underuse, max framerate should go back to codec configured fps.
3766 // Set extra low fps, to make sure it's actually reset, not just incremented.
3767 stats = stats_proxy_->GetStats();
3768 stats.input_frame_rate = adapted_framerate / 2;
3769 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003770 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003771 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3772 EXPECT_EQ(
3773 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3774 kFramerate);
3775
3776 video_stream_encoder_->Stop();
3777}
3778
3779TEST_F(VideoStreamEncoderTest,
3780 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3781 const int kFrameWidth = 1280;
3782 const int kFrameHeight = 720;
3783 const int kLowFramerate = 15;
3784 const int kHighFramerate = 25;
3785
Henrik Boström381d1092020-05-12 18:49:07 +02003786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003787 DataRate::BitsPerSec(kTargetBitrateBps),
3788 DataRate::BitsPerSec(kTargetBitrateBps),
3789 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003790 test::FrameForwarder source;
3791 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003792 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003793
3794 // Trigger initial configuration.
3795 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003796 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003797 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3798 video_encoder_config.number_of_streams = 1;
3799 video_encoder_config.video_stream_factory =
3800 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3801 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3802 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003803 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003804 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3805
3806 EXPECT_EQ(
3807 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3808 kLowFramerate);
3809
3810 // Trigger overuse, max framerate should be reduced.
3811 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3812 stats.input_frame_rate = kLowFramerate;
3813 stats_proxy_->SetMockStats(stats);
3814 video_stream_encoder_->TriggerCpuOveruse();
3815 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3816 int adapted_framerate =
3817 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3818 EXPECT_LT(adapted_framerate, kLowFramerate);
3819
3820 // Reconfigure the encoder with a new (higher max framerate), max fps should
3821 // still respect the adaptation.
3822 video_encoder_config.video_stream_factory =
3823 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3824 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3825 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003826 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003827 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3828
3829 EXPECT_EQ(
3830 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3831 adapted_framerate);
3832
3833 // Trigger underuse, max framerate should go back to codec configured fps.
3834 stats = stats_proxy_->GetStats();
3835 stats.input_frame_rate = adapted_framerate;
3836 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003837 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003838 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3839 EXPECT_EQ(
3840 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3841 kHighFramerate);
3842
3843 video_stream_encoder_->Stop();
3844}
3845
mflodmancc3d4422017-08-03 08:27:51 -07003846TEST_F(VideoStreamEncoderTest,
3847 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003848 const int kFrameWidth = 1280;
3849 const int kFrameHeight = 720;
3850 const int kFramerate = 24;
3851
Henrik Boström381d1092020-05-12 18:49:07 +02003852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003853 DataRate::BitsPerSec(kTargetBitrateBps),
3854 DataRate::BitsPerSec(kTargetBitrateBps),
3855 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003856 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003857 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003858 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003859
3860 // Trigger initial configuration.
3861 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003862 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003863 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3864 video_encoder_config.number_of_streams = 1;
3865 video_encoder_config.video_stream_factory =
3866 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3867 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003868 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003869 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003870 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003871
Niels Möller7dc26b72017-12-06 10:27:48 +01003872 EXPECT_EQ(
3873 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3874 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003875
3876 // Trigger overuse, max framerate should be reduced.
3877 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3878 stats.input_frame_rate = kFramerate;
3879 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003880 video_stream_encoder_->TriggerCpuOveruse();
3881 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003882 int adapted_framerate =
3883 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003884 EXPECT_LT(adapted_framerate, kFramerate);
3885
3886 // Change degradation preference to not enable framerate scaling. Target
3887 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02003888 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003889 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01003890 EXPECT_EQ(
3891 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3892 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003893
mflodmancc3d4422017-08-03 08:27:51 -07003894 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003895}
3896
mflodmancc3d4422017-08-03 08:27:51 -07003897TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003898 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02003899 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003900 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3901 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3902 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003903 const int kWidth = 640;
3904 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003905
asaperssonfab67072017-04-04 05:51:49 -07003906 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003907
3908 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003909 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003910
3911 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003912 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003913
sprangc5d62e22017-04-02 23:53:04 -07003914 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003915
asaperssonfab67072017-04-04 05:51:49 -07003916 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003917 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003918 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003919
3920 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003921 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003922
sprangc5d62e22017-04-02 23:53:04 -07003923 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003924
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003926}
3927
mflodmancc3d4422017-08-03 08:27:51 -07003928TEST_F(VideoStreamEncoderTest,
3929 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003930 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02003931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003932 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3933 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3934 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003935 const int kWidth = 640;
3936 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003937
3938 // We expect the n initial frames to get dropped.
3939 int i;
3940 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003941 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003942 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003943 }
3944 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003945 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003946 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003947
3948 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003949 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003950
mflodmancc3d4422017-08-03 08:27:51 -07003951 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003952}
3953
mflodmancc3d4422017-08-03 08:27:51 -07003954TEST_F(VideoStreamEncoderTest,
3955 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003956 const int kWidth = 640;
3957 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02003958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003959 DataRate::BitsPerSec(kLowTargetBitrateBps),
3960 DataRate::BitsPerSec(kLowTargetBitrateBps),
3961 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003962
3963 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003964 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003965 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003966
asaperssonfab67072017-04-04 05:51:49 -07003967 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003968 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003969 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003970
mflodmancc3d4422017-08-03 08:27:51 -07003971 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003972}
3973
mflodmancc3d4422017-08-03 08:27:51 -07003974TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003975 const int kWidth = 640;
3976 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003977 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003978
3979 VideoEncoderConfig video_encoder_config;
3980 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3981 // Make format different, to force recreation of encoder.
3982 video_encoder_config.video_format.parameters["foo"] = "foo";
3983 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003984 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02003985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003986 DataRate::BitsPerSec(kLowTargetBitrateBps),
3987 DataRate::BitsPerSec(kLowTargetBitrateBps),
3988 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003989
kthelgasonb83797b2017-02-14 11:57:25 -08003990 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003991 video_stream_encoder_->SetSource(&video_source_,
3992 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003993
asaperssonfab67072017-04-04 05:51:49 -07003994 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003995 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003996 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003997
mflodmancc3d4422017-08-03 08:27:51 -07003998 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003999 fake_encoder_.SetQualityScaling(true);
4000}
4001
Åsa Persson139f4dc2019-08-02 09:29:58 +02004002TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4003 webrtc::test::ScopedFieldTrials field_trials(
4004 "WebRTC-Video-QualityScalerSettings/"
4005 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4006 // Reset encoder for field trials to take effect.
4007 ConfigureEncoder(video_encoder_config_.Copy());
4008 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4009 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4010 const int kWidth = 640;
4011 const int kHeight = 360;
4012
Henrik Boström381d1092020-05-12 18:49:07 +02004013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004014 DataRate::BitsPerSec(kTargetBitrateBps),
4015 DataRate::BitsPerSec(kTargetBitrateBps),
4016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004017 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4018 // Frame should not be dropped.
4019 WaitForEncodedFrame(1);
4020
Henrik Boström381d1092020-05-12 18:49:07 +02004021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004022 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4023 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4024 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004025 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4026 // Frame should not be dropped.
4027 WaitForEncodedFrame(2);
4028
Henrik Boström381d1092020-05-12 18:49:07 +02004029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004030 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4031 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4032 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004033 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4034 // Expect to drop this frame, the wait should time out.
4035 ExpectDroppedFrame();
4036
4037 // Expect the sink_wants to specify a scaled frame.
4038 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
4039 video_stream_encoder_->Stop();
4040}
4041
Åsa Perssone644a032019-11-08 15:56:00 +01004042TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4043 webrtc::test::ScopedFieldTrials field_trials(
4044 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4045
4046 // Reset encoder for field trials to take effect.
4047 VideoEncoderConfig config = video_encoder_config_.Copy();
4048 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004049 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004050 ConfigureEncoder(std::move(config));
4051 fake_encoder_.SetQp(kQpLow);
4052
4053 // Enable MAINTAIN_FRAMERATE preference.
4054 AdaptingFrameForwarder source;
4055 source.set_adaptation_enabled(true);
4056 video_stream_encoder_->SetSource(&source,
4057 DegradationPreference::MAINTAIN_FRAMERATE);
4058
4059 // Start at low bitrate.
4060 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4062 DataRate::BitsPerSec(kLowBitrateBps),
4063 DataRate::BitsPerSec(kLowBitrateBps),
4064 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004065
4066 // Expect first frame to be dropped and resolution to be limited.
4067 const int kWidth = 1280;
4068 const int kHeight = 720;
4069 const int64_t kFrameIntervalMs = 100;
4070 int64_t timestamp_ms = kFrameIntervalMs;
4071 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4072 ExpectDroppedFrame();
4073 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4074
4075 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4077 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004078
4079 // Insert frames and advance |min_duration_ms|.
4080 for (size_t i = 1; i <= 10; i++) {
4081 timestamp_ms += kFrameIntervalMs;
4082 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4083 WaitForEncodedFrame(timestamp_ms);
4084 }
4085 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4086 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4087
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004088 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004089
4090 // Insert frame should trigger high BW and release quality limitation.
4091 timestamp_ms += kFrameIntervalMs;
4092 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4093 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004094 // The ramp-up code involves the adaptation queue, give it time to execute.
4095 // TODO(hbos): Can we await an appropriate event instead?
4096 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Åsa Perssone644a032019-11-08 15:56:00 +01004097 VerifyFpsMaxResolutionMax(source.sink_wants());
4098
4099 // Frame should not be adapted.
4100 timestamp_ms += kFrameIntervalMs;
4101 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4102 WaitForEncodedFrame(kWidth, kHeight);
4103 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4104
4105 video_stream_encoder_->Stop();
4106}
4107
mflodmancc3d4422017-08-03 08:27:51 -07004108TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004109 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4110 const int kTooSmallWidth = 10;
4111 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004112 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004113 DataRate::BitsPerSec(kTargetBitrateBps),
4114 DataRate::BitsPerSec(kTargetBitrateBps),
4115 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004116
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004117 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004118 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004119 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004120 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004121 VerifyNoLimitation(source.sink_wants());
4122 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4123
4124 // Trigger adapt down, too small frame, expect no change.
4125 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004126 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004127 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004128 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07004129 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4130 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4131
mflodmancc3d4422017-08-03 08:27:51 -07004132 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004133}
4134
mflodmancc3d4422017-08-03 08:27:51 -07004135TEST_F(VideoStreamEncoderTest,
4136 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004137 const int kTooSmallWidth = 10;
4138 const int kTooSmallHeight = 10;
4139 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004140 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004141 DataRate::BitsPerSec(kTargetBitrateBps),
4142 DataRate::BitsPerSec(kTargetBitrateBps),
4143 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004144
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004145 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004146 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004147 video_stream_encoder_->SetSource(&source,
4148 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004149 VerifyNoLimitation(source.sink_wants());
4150 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4151 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4152
4153 // Trigger adapt down, expect limited framerate.
4154 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004155 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004156 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004157 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4158 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4159 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4160 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4161
4162 // Trigger adapt down, too small frame, expect no change.
4163 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004164 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004165 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004166 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4167 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4168 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4169 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4170
mflodmancc3d4422017-08-03 08:27:51 -07004171 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004172}
4173
mflodmancc3d4422017-08-03 08:27:51 -07004174TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004175 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004177 DataRate::BitsPerSec(kTargetBitrateBps),
4178 DataRate::BitsPerSec(kTargetBitrateBps),
4179 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004180 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004181 const int kFrameWidth = 1280;
4182 const int kFrameHeight = 720;
4183 video_source_.IncomingCapturedFrame(
4184 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004185 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004186 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004187}
4188
sprangb1ca0732017-02-01 08:38:12 -08004189// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004190TEST_F(VideoStreamEncoderTest,
4191 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004192 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004193 DataRate::BitsPerSec(kTargetBitrateBps),
4194 DataRate::BitsPerSec(kTargetBitrateBps),
4195 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004196
4197 const int kFrameWidth = 1280;
4198 const int kFrameHeight = 720;
4199 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004200 // requested by
4201 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004202 video_source_.set_adaptation_enabled(true);
4203
4204 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004205 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004206 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004207
4208 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004209 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004210 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004211 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004212 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004213
asaperssonfab67072017-04-04 05:51:49 -07004214 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004215 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004216 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004217 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004218 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004219
mflodmancc3d4422017-08-03 08:27:51 -07004220 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004221}
sprangfe627f32017-03-29 08:24:59 -07004222
mflodmancc3d4422017-08-03 08:27:51 -07004223TEST_F(VideoStreamEncoderTest,
4224 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004225 const int kFrameWidth = 1280;
4226 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004227
Henrik Boström381d1092020-05-12 18:49:07 +02004228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004229 DataRate::BitsPerSec(kTargetBitrateBps),
4230 DataRate::BitsPerSec(kTargetBitrateBps),
4231 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004232 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004233 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004234 video_source_.set_adaptation_enabled(true);
4235
sprang4847ae62017-06-27 07:06:52 -07004236 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004237
4238 video_source_.IncomingCapturedFrame(
4239 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004240 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004241
4242 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004243 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004244
4245 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004246 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004247 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004248 video_source_.IncomingCapturedFrame(
4249 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004250 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004251 }
4252
4253 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004254 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004255 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004256 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004257 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004258 video_source_.IncomingCapturedFrame(
4259 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004260 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004261 ++num_frames_dropped;
4262 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004263 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004264 }
4265 }
4266
sprang4847ae62017-06-27 07:06:52 -07004267 // Add some slack to account for frames dropped by the frame dropper.
4268 const int kErrorMargin = 1;
4269 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004270 kErrorMargin);
4271
4272 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004273 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004274 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004275 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004276 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004277 video_source_.IncomingCapturedFrame(
4278 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004279 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004280 ++num_frames_dropped;
4281 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004282 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004283 }
4284 }
sprang4847ae62017-06-27 07:06:52 -07004285 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004286 kErrorMargin);
4287
4288 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004289 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004290 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004291 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004292 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004293 video_source_.IncomingCapturedFrame(
4294 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004295 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004296 ++num_frames_dropped;
4297 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004298 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004299 }
4300 }
sprang4847ae62017-06-27 07:06:52 -07004301 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004302 kErrorMargin);
4303
4304 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004305 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004306 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004307 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004308 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004309 video_source_.IncomingCapturedFrame(
4310 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004311 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004312 ++num_frames_dropped;
4313 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004314 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004315 }
4316 }
4317 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4318
mflodmancc3d4422017-08-03 08:27:51 -07004319 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004320}
4321
mflodmancc3d4422017-08-03 08:27:51 -07004322TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004323 const int kFramerateFps = 5;
4324 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004325 const int kFrameWidth = 1280;
4326 const int kFrameHeight = 720;
4327
sprang4847ae62017-06-27 07:06:52 -07004328 // Reconfigure encoder with two temporal layers and screensharing, which will
4329 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004330 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004331
Henrik Boström381d1092020-05-12 18:49:07 +02004332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004333 DataRate::BitsPerSec(kTargetBitrateBps),
4334 DataRate::BitsPerSec(kTargetBitrateBps),
4335 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004336 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004337 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004338 video_source_.set_adaptation_enabled(true);
4339
sprang4847ae62017-06-27 07:06:52 -07004340 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004341
4342 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004343 rtc::VideoSinkWants last_wants;
4344 do {
4345 last_wants = video_source_.sink_wants();
4346
sprangc5d62e22017-04-02 23:53:04 -07004347 // Insert frames to get a new fps estimate...
4348 for (int j = 0; j < kFramerateFps; ++j) {
4349 video_source_.IncomingCapturedFrame(
4350 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004351 if (video_source_.last_sent_width()) {
4352 sink_.WaitForEncodedFrame(timestamp_ms);
4353 }
sprangc5d62e22017-04-02 23:53:04 -07004354 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004355 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004356 }
4357 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004358 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004359 } while (video_source_.sink_wants().max_framerate_fps <
4360 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004361
Jonathan Yubc771b72017-12-08 17:04:29 -08004362 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004363
mflodmancc3d4422017-08-03 08:27:51 -07004364 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004365}
asaperssonf7e294d2017-06-13 23:25:22 -07004366
mflodmancc3d4422017-08-03 08:27:51 -07004367TEST_F(VideoStreamEncoderTest,
4368 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004369 const int kWidth = 1280;
4370 const int kHeight = 720;
4371 const int64_t kFrameIntervalMs = 150;
4372 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004374 DataRate::BitsPerSec(kTargetBitrateBps),
4375 DataRate::BitsPerSec(kTargetBitrateBps),
4376 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004377
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004378 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004379 AdaptingFrameForwarder source;
4380 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004381 video_stream_encoder_->SetSource(&source,
4382 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004383 timestamp_ms += kFrameIntervalMs;
4384 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004385 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004386 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004387 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4388 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4389 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
4391 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004392 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004393 timestamp_ms += kFrameIntervalMs;
4394 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004395 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004396 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4397 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4398 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4399 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4400
4401 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004402 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004403 timestamp_ms += kFrameIntervalMs;
4404 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004405 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004406 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4407 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4408 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4409 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4410
4411 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004412 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004413 timestamp_ms += kFrameIntervalMs;
4414 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004415 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004416 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4417 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4418 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4419 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4420
4421 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004422 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004423 timestamp_ms += kFrameIntervalMs;
4424 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004425 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004426 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4427 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4428 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4429 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4430
4431 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004432 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004433 timestamp_ms += kFrameIntervalMs;
4434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004435 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004436 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4437 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4439 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4440
4441 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004442 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004443 timestamp_ms += kFrameIntervalMs;
4444 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004445 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004446 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4447 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4448 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4449 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4450
4451 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004452 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004453 timestamp_ms += kFrameIntervalMs;
4454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004455 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004456 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4457 rtc::VideoSinkWants last_wants = source.sink_wants();
4458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4460 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
4462 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004463 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004466 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004467 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4468 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4470 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4471
4472 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004473 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004474 timestamp_ms += kFrameIntervalMs;
4475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004476 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004477 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4479 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4480 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4481
4482 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004483 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004484 timestamp_ms += kFrameIntervalMs;
4485 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004486 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004487 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4490 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4491
4492 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004493 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004494 timestamp_ms += kFrameIntervalMs;
4495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004496 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004497 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4499 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4500 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4501
4502 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004503 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004504 timestamp_ms += kFrameIntervalMs;
4505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004506 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004507 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4509 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4510 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4511
4512 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004513 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004514 timestamp_ms += kFrameIntervalMs;
4515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004516 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004517 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4520 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4521
4522 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004523 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004524 timestamp_ms += kFrameIntervalMs;
4525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004526 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004527 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4530 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4531
Åsa Persson30ab0152019-08-27 12:22:33 +02004532 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004533 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004534 timestamp_ms += kFrameIntervalMs;
4535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004536 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004537 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004538 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004539 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4541 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4542
4543 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004544 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004545 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004546 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4547
mflodmancc3d4422017-08-03 08:27:51 -07004548 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004549}
4550
mflodmancc3d4422017-08-03 08:27:51 -07004551TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004552 const int kWidth = 1280;
4553 const int kHeight = 720;
4554 const int64_t kFrameIntervalMs = 150;
4555 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004557 DataRate::BitsPerSec(kTargetBitrateBps),
4558 DataRate::BitsPerSec(kTargetBitrateBps),
4559 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004560
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004561 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004562 AdaptingFrameForwarder source;
4563 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004564 video_stream_encoder_->SetSource(&source,
4565 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004566 timestamp_ms += kFrameIntervalMs;
4567 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004568 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004569 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4571 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4572 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4573 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4574 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4575 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4576
4577 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004578 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004579 timestamp_ms += kFrameIntervalMs;
4580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004581 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004582 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4583 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4584 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4585 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4586 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4587 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4588 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4589
4590 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004591 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004592 timestamp_ms += kFrameIntervalMs;
4593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004594 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004595 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4598 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4600 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4601 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4602
4603 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004604 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004605 timestamp_ms += kFrameIntervalMs;
4606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004607 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004608 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4611 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4612 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4613 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4614 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4615
4616 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004617 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004618 timestamp_ms += kFrameIntervalMs;
4619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004620 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004621 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4624 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4625 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4626 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4627 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4628
4629 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004630 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004631 timestamp_ms += kFrameIntervalMs;
4632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004633 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004634 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4635 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4636 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4637 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4639 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4640 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4641
4642 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004643 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004644 timestamp_ms += kFrameIntervalMs;
4645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004646 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004647 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004648 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4653 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4654 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4655
4656 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004657 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004658 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004659 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4660 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4661
mflodmancc3d4422017-08-03 08:27:51 -07004662 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004663}
4664
mflodmancc3d4422017-08-03 08:27:51 -07004665TEST_F(VideoStreamEncoderTest,
4666 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004667 const int kWidth = 640;
4668 const int kHeight = 360;
4669 const int kFpsLimit = 15;
4670 const int64_t kFrameIntervalMs = 150;
4671 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004673 DataRate::BitsPerSec(kTargetBitrateBps),
4674 DataRate::BitsPerSec(kTargetBitrateBps),
4675 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004676
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004677 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004678 AdaptingFrameForwarder source;
4679 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004680 video_stream_encoder_->SetSource(&source,
4681 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004682 timestamp_ms += kFrameIntervalMs;
4683 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004684 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004685 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004686 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4688 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4689 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4690 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4691 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4692
4693 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004694 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004695 timestamp_ms += kFrameIntervalMs;
4696 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004697 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004698 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4699 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4700 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4701 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4702 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4703 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4704 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4705
4706 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004707 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004708 timestamp_ms += kFrameIntervalMs;
4709 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004710 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004711 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4712 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4713 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4714 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4715 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4716 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4717 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4718
4719 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004720 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004721 timestamp_ms += kFrameIntervalMs;
4722 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004723 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004724 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4725 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4726 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4727 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4728 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4729 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4730 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4731
4732 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004733 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004734 timestamp_ms += kFrameIntervalMs;
4735 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004736 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004737 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004738 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4739 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4740 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4741 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4742 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4743 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4744
4745 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004746 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004747 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004748 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4749 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4750
mflodmancc3d4422017-08-03 08:27:51 -07004751 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004752}
4753
mflodmancc3d4422017-08-03 08:27:51 -07004754TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004755 const int kFrameWidth = 1920;
4756 const int kFrameHeight = 1080;
4757 // 3/4 of 1920.
4758 const int kAdaptedFrameWidth = 1440;
4759 // 3/4 of 1080 rounded down to multiple of 4.
4760 const int kAdaptedFrameHeight = 808;
4761 const int kFramerate = 24;
4762
Henrik Boström381d1092020-05-12 18:49:07 +02004763 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004764 DataRate::BitsPerSec(kTargetBitrateBps),
4765 DataRate::BitsPerSec(kTargetBitrateBps),
4766 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004767 // Trigger reconfigure encoder (without resetting the entire instance).
4768 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004769 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004770 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4771 video_encoder_config.number_of_streams = 1;
4772 video_encoder_config.video_stream_factory =
4773 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004774 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004775 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004777
4778 video_source_.set_adaptation_enabled(true);
4779
4780 video_source_.IncomingCapturedFrame(
4781 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004782 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004783
4784 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004785 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004786 video_source_.IncomingCapturedFrame(
4787 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004788 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004789
mflodmancc3d4422017-08-03 08:27:51 -07004790 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004791}
4792
mflodmancc3d4422017-08-03 08:27:51 -07004793TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004794 const int kFrameWidth = 1280;
4795 const int kFrameHeight = 720;
4796 const int kLowFps = 2;
4797 const int kHighFps = 30;
4798
Henrik Boström381d1092020-05-12 18:49:07 +02004799 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004800 DataRate::BitsPerSec(kTargetBitrateBps),
4801 DataRate::BitsPerSec(kTargetBitrateBps),
4802 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004803
4804 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4805 max_framerate_ = kLowFps;
4806
4807 // Insert 2 seconds of 2fps video.
4808 for (int i = 0; i < kLowFps * 2; ++i) {
4809 video_source_.IncomingCapturedFrame(
4810 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4811 WaitForEncodedFrame(timestamp_ms);
4812 timestamp_ms += 1000 / kLowFps;
4813 }
4814
4815 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004817 DataRate::BitsPerSec(kTargetBitrateBps),
4818 DataRate::BitsPerSec(kTargetBitrateBps),
4819 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004820 video_source_.IncomingCapturedFrame(
4821 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4822 WaitForEncodedFrame(timestamp_ms);
4823 timestamp_ms += 1000 / kLowFps;
4824
4825 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4826
4827 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004828 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004829 const int kFrameIntervalMs = 1000 / kHighFps;
4830 max_framerate_ = kHighFps;
4831 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4832 video_source_.IncomingCapturedFrame(
4833 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4834 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4835 // be dropped if the encoder hans't been updated with the new higher target
4836 // framerate yet, causing it to overshoot the target bitrate and then
4837 // suffering the wrath of the media optimizer.
4838 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4839 timestamp_ms += kFrameIntervalMs;
4840 }
4841
4842 // Don expect correct measurement just yet, but it should be higher than
4843 // before.
4844 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4845
mflodmancc3d4422017-08-03 08:27:51 -07004846 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004847}
4848
mflodmancc3d4422017-08-03 08:27:51 -07004849TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004850 const int kFrameWidth = 1280;
4851 const int kFrameHeight = 720;
4852 const int kTargetBitrateBps = 1000000;
4853
4854 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004855 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02004856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004857 DataRate::BitsPerSec(kTargetBitrateBps),
4858 DataRate::BitsPerSec(kTargetBitrateBps),
4859 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004860 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004861
4862 // Insert a first video frame, causes another bitrate update.
4863 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4864 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4865 video_source_.IncomingCapturedFrame(
4866 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4867 WaitForEncodedFrame(timestamp_ms);
4868
4869 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02004870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4871 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
4872 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004873
4874 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004875 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004876 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004877
4878 // Bitrate observer should not be called.
4879 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4880 video_source_.IncomingCapturedFrame(
4881 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4882 ExpectDroppedFrame();
4883
mflodmancc3d4422017-08-03 08:27:51 -07004884 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004885}
ilnik6b826ef2017-06-16 06:53:48 -07004886
Niels Möller4db138e2018-04-19 09:04:13 +02004887TEST_F(VideoStreamEncoderTest,
4888 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4889 const int kFrameWidth = 1280;
4890 const int kFrameHeight = 720;
4891 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02004892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004893 DataRate::BitsPerSec(kTargetBitrateBps),
4894 DataRate::BitsPerSec(kTargetBitrateBps),
4895 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004896 video_source_.IncomingCapturedFrame(
4897 CreateFrame(1, kFrameWidth, kFrameHeight));
4898 WaitForEncodedFrame(1);
4899 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4900 .low_encode_usage_threshold_percent,
4901 default_options.low_encode_usage_threshold_percent);
4902 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4903 .high_encode_usage_threshold_percent,
4904 default_options.high_encode_usage_threshold_percent);
4905 video_stream_encoder_->Stop();
4906}
4907
4908TEST_F(VideoStreamEncoderTest,
4909 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4910 const int kFrameWidth = 1280;
4911 const int kFrameHeight = 720;
4912 CpuOveruseOptions hardware_options;
4913 hardware_options.low_encode_usage_threshold_percent = 150;
4914 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004915 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004916
Henrik Boström381d1092020-05-12 18:49:07 +02004917 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004918 DataRate::BitsPerSec(kTargetBitrateBps),
4919 DataRate::BitsPerSec(kTargetBitrateBps),
4920 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004921 video_source_.IncomingCapturedFrame(
4922 CreateFrame(1, kFrameWidth, kFrameHeight));
4923 WaitForEncodedFrame(1);
4924 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4925 .low_encode_usage_threshold_percent,
4926 hardware_options.low_encode_usage_threshold_percent);
4927 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4928 .high_encode_usage_threshold_percent,
4929 hardware_options.high_encode_usage_threshold_percent);
4930 video_stream_encoder_->Stop();
4931}
4932
Niels Möller6bb5ab92019-01-11 11:11:10 +01004933TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4934 const int kFrameWidth = 320;
4935 const int kFrameHeight = 240;
4936 const int kFps = 30;
4937 const int kTargetBitrateBps = 120000;
4938 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4939
Henrik Boström381d1092020-05-12 18:49:07 +02004940 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004941 DataRate::BitsPerSec(kTargetBitrateBps),
4942 DataRate::BitsPerSec(kTargetBitrateBps),
4943 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004944
4945 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4946 max_framerate_ = kFps;
4947
4948 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4949 fake_encoder_.SimulateOvershoot(1.0);
4950 int num_dropped = 0;
4951 for (int i = 0; i < kNumFramesInRun; ++i) {
4952 video_source_.IncomingCapturedFrame(
4953 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4954 // Wait up to two frame durations for a frame to arrive.
4955 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4956 ++num_dropped;
4957 }
4958 timestamp_ms += 1000 / kFps;
4959 }
4960
Erik Språnga8d48ab2019-02-08 14:17:40 +01004961 // Framerate should be measured to be near the expected target rate.
4962 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4963
4964 // Frame drops should be within 5% of expected 0%.
4965 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004966
4967 // Make encoder produce frames at double the expected bitrate during 3 seconds
4968 // of video, verify number of drops. Rate needs to be slightly changed in
4969 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004970 double overshoot_factor = 2.0;
4971 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4972 // With bitrate adjuster, when need to overshoot even more to trigger
4973 // frame dropping.
4974 overshoot_factor *= 2;
4975 }
4976 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02004977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004978 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4979 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4980 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004981 num_dropped = 0;
4982 for (int i = 0; i < kNumFramesInRun; ++i) {
4983 video_source_.IncomingCapturedFrame(
4984 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4985 // Wait up to two frame durations for a frame to arrive.
4986 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4987 ++num_dropped;
4988 }
4989 timestamp_ms += 1000 / kFps;
4990 }
4991
Henrik Boström381d1092020-05-12 18:49:07 +02004992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004993 DataRate::BitsPerSec(kTargetBitrateBps),
4994 DataRate::BitsPerSec(kTargetBitrateBps),
4995 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004996
4997 // Target framerate should be still be near the expected target, despite
4998 // the frame drops.
4999 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5000
5001 // Frame drops should be within 5% of expected 50%.
5002 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005003
5004 video_stream_encoder_->Stop();
5005}
5006
5007TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5008 const int kFrameWidth = 320;
5009 const int kFrameHeight = 240;
5010 const int kActualInputFps = 24;
5011 const int kTargetBitrateBps = 120000;
5012
5013 ASSERT_GT(max_framerate_, kActualInputFps);
5014
5015 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5016 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005018 DataRate::BitsPerSec(kTargetBitrateBps),
5019 DataRate::BitsPerSec(kTargetBitrateBps),
5020 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005021
5022 // Insert 3 seconds of video, with an input fps lower than configured max.
5023 for (int i = 0; i < kActualInputFps * 3; ++i) {
5024 video_source_.IncomingCapturedFrame(
5025 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5026 // Wait up to two frame durations for a frame to arrive.
5027 WaitForEncodedFrame(timestamp_ms);
5028 timestamp_ms += 1000 / kActualInputFps;
5029 }
5030
5031 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5032
5033 video_stream_encoder_->Stop();
5034}
5035
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005036TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5037 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005039 DataRate::BitsPerSec(kTargetBitrateBps),
5040 DataRate::BitsPerSec(kTargetBitrateBps),
5041 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005042
5043 fake_encoder_.BlockNextEncode();
5044 video_source_.IncomingCapturedFrame(
5045 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5046 WaitForEncodedFrame(1);
5047 // On the very first frame full update should be forced.
5048 rect = fake_encoder_.GetLastUpdateRect();
5049 EXPECT_EQ(rect.offset_x, 0);
5050 EXPECT_EQ(rect.offset_y, 0);
5051 EXPECT_EQ(rect.height, codec_height_);
5052 EXPECT_EQ(rect.width, codec_width_);
5053 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5054 // call to ContinueEncode.
5055 video_source_.IncomingCapturedFrame(
5056 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5057 ExpectDroppedFrame();
5058 video_source_.IncomingCapturedFrame(
5059 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5060 ExpectDroppedFrame();
5061 fake_encoder_.ContinueEncode();
5062 WaitForEncodedFrame(3);
5063 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5064 rect = fake_encoder_.GetLastUpdateRect();
5065 EXPECT_EQ(rect.offset_x, 1);
5066 EXPECT_EQ(rect.offset_y, 0);
5067 EXPECT_EQ(rect.width, 10);
5068 EXPECT_EQ(rect.height, 1);
5069
5070 video_source_.IncomingCapturedFrame(
5071 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5072 WaitForEncodedFrame(4);
5073 // Previous frame was encoded, so no accumulation should happen.
5074 rect = fake_encoder_.GetLastUpdateRect();
5075 EXPECT_EQ(rect.offset_x, 0);
5076 EXPECT_EQ(rect.offset_y, 0);
5077 EXPECT_EQ(rect.width, 1);
5078 EXPECT_EQ(rect.height, 1);
5079
5080 video_stream_encoder_->Stop();
5081}
5082
Erik Språngd7329ca2019-02-21 21:19:53 +01005083TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005084 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005085 DataRate::BitsPerSec(kTargetBitrateBps),
5086 DataRate::BitsPerSec(kTargetBitrateBps),
5087 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005088
5089 // First frame is always keyframe.
5090 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5091 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005092 EXPECT_THAT(
5093 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005094 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005095
5096 // Insert delta frame.
5097 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5098 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005099 EXPECT_THAT(
5100 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005101 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005102
5103 // Request next frame be a key-frame.
5104 video_stream_encoder_->SendKeyFrame();
5105 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5106 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005107 EXPECT_THAT(
5108 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005109 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005110
5111 video_stream_encoder_->Stop();
5112}
5113
5114TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5115 // Setup simulcast with three streams.
5116 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005118 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5119 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5120 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005121 // Wait for all three layers before triggering event.
5122 sink_.SetNumExpectedLayers(3);
5123
5124 // First frame is always keyframe.
5125 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5126 WaitForEncodedFrame(1);
5127 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005128 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5129 VideoFrameType::kVideoFrameKey,
5130 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005131
5132 // Insert delta frame.
5133 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5134 WaitForEncodedFrame(2);
5135 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005136 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5137 VideoFrameType::kVideoFrameDelta,
5138 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005139
5140 // Request next frame be a key-frame.
5141 // Only first stream is configured to produce key-frame.
5142 video_stream_encoder_->SendKeyFrame();
5143 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5144 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005145
5146 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5147 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005148 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005149 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005150 VideoFrameType::kVideoFrameKey,
5151 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005152
5153 video_stream_encoder_->Stop();
5154}
5155
5156TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5157 // Configure internal source factory and setup test again.
5158 encoder_factory_.SetHasInternalSource(true);
5159 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005161 DataRate::BitsPerSec(kTargetBitrateBps),
5162 DataRate::BitsPerSec(kTargetBitrateBps),
5163 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005164
5165 // Call encoder directly, simulating internal source where encoded frame
5166 // callback in VideoStreamEncoder is called despite no OnFrame().
5167 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5168 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005169 EXPECT_THAT(
5170 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005171 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005172
Niels Möller8f7ce222019-03-21 15:43:58 +01005173 const std::vector<VideoFrameType> kDeltaFrame = {
5174 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005175 // Need to set timestamp manually since manually for injected frame.
5176 VideoFrame frame = CreateFrame(101, nullptr);
5177 frame.set_timestamp(101);
5178 fake_encoder_.InjectFrame(frame, false);
5179 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005180 EXPECT_THAT(
5181 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005182 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005183
5184 // Request key-frame. The forces a dummy frame down into the encoder.
5185 fake_encoder_.ExpectNullFrame();
5186 video_stream_encoder_->SendKeyFrame();
5187 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005188 EXPECT_THAT(
5189 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005190 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005191
5192 video_stream_encoder_->Stop();
5193}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005194
5195TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5196 // Configure internal source factory and setup test again.
5197 encoder_factory_.SetHasInternalSource(true);
5198 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005199 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005200 DataRate::BitsPerSec(kTargetBitrateBps),
5201 DataRate::BitsPerSec(kTargetBitrateBps),
5202 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005203
5204 int64_t timestamp = 1;
5205 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005206 image.SetEncodedData(
5207 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005208 image.capture_time_ms_ = ++timestamp;
5209 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5210 const int64_t kEncodeFinishDelayMs = 10;
5211 image.timing_.encode_start_ms = timestamp;
5212 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5213 fake_encoder_.InjectEncodedImage(image);
5214 // Wait for frame without incrementing clock.
5215 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5216 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5217 // capture timestamp should be kEncodeFinishDelayMs in the past.
5218 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5219 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5220 kEncodeFinishDelayMs);
5221
5222 video_stream_encoder_->Stop();
5223}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005224
5225TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5226 // Configure internal source factory and setup test again.
5227 encoder_factory_.SetHasInternalSource(true);
5228 ResetEncoder("H264", 1, 1, 1, false);
5229
5230 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5231 image._frameType = VideoFrameType::kVideoFrameKey;
5232
5233 CodecSpecificInfo codec_specific_info;
5234 codec_specific_info.codecType = kVideoCodecH264;
5235
5236 RTPFragmentationHeader fragmentation;
5237 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5238 fragmentation.fragmentationOffset[0] = 4;
5239 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5240
5241 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5242 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5243
5244 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5245 testing::ElementsAreArray(optimal_sps));
5246 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5247 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5248 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5249 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5250
5251 video_stream_encoder_->Stop();
5252}
5253
5254TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5255 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5256 0x00, 0x00, 0x03, 0x03, 0xF4,
5257 0x05, 0x03, 0xC7, 0xC0};
5258
5259 // Configure internal source factory and setup test again.
5260 encoder_factory_.SetHasInternalSource(true);
5261 ResetEncoder("H264", 1, 1, 1, false);
5262
5263 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5264 image._frameType = VideoFrameType::kVideoFrameKey;
5265
5266 CodecSpecificInfo codec_specific_info;
5267 codec_specific_info.codecType = kVideoCodecH264;
5268
5269 RTPFragmentationHeader fragmentation;
5270 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5271 fragmentation.fragmentationOffset[0] = 4;
5272 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5273
5274 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5275 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5276
5277 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5278 testing::ElementsAreArray(optimal_sps));
5279 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5280 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5281 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5282 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5283
5284 video_stream_encoder_->Stop();
5285}
5286
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005287TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5288 const int kFrameWidth = 1280;
5289 const int kFrameHeight = 720;
5290 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5291
Henrik Boström381d1092020-05-12 18:49:07 +02005292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005293 DataRate::BitsPerSec(kTargetBitrateBps),
5294 DataRate::BitsPerSec(kTargetBitrateBps),
5295 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5297
5298 // Insert a first video frame. It should be dropped because of downscale in
5299 // resolution.
5300 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5301 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5302 frame.set_rotation(kVideoRotation_270);
5303 video_source_.IncomingCapturedFrame(frame);
5304
5305 ExpectDroppedFrame();
5306
5307 // Second frame is downscaled.
5308 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5309 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5310 frame.set_rotation(kVideoRotation_90);
5311 video_source_.IncomingCapturedFrame(frame);
5312
5313 WaitForEncodedFrame(timestamp_ms);
5314 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5315
5316 // Insert another frame, also downscaled.
5317 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5318 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5319 frame.set_rotation(kVideoRotation_180);
5320 video_source_.IncomingCapturedFrame(frame);
5321
5322 WaitForEncodedFrame(timestamp_ms);
5323 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5324
5325 video_stream_encoder_->Stop();
5326}
5327
Erik Språng5056af02019-09-02 15:53:11 +02005328TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5329 const int kFrameWidth = 320;
5330 const int kFrameHeight = 180;
5331
5332 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005334 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5335 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5336 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005337 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005338 /*rtt_ms=*/0,
5339 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005340
5341 // Insert a first video frame so that encoder gets configured.
5342 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5343 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5344 frame.set_rotation(kVideoRotation_270);
5345 video_source_.IncomingCapturedFrame(frame);
5346 WaitForEncodedFrame(timestamp_ms);
5347
5348 // Set a target rate below the minimum allowed by the codec settings.
5349 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005350 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5351 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005353 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005354 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005355 /*link_allocation=*/target_rate,
5356 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005357 /*rtt_ms=*/0,
5358 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5360
5361 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5362 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5363 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005364 DataRate allocation_sum =
5365 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005366 EXPECT_EQ(min_rate, allocation_sum);
5367 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5368
5369 video_stream_encoder_->Stop();
5370}
5371
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005372TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005374 DataRate::BitsPerSec(kTargetBitrateBps),
5375 DataRate::BitsPerSec(kTargetBitrateBps),
5376 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005377 // Capture a frame and wait for it to synchronize with the encoder thread.
5378 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5379 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5380 WaitForEncodedFrame(1);
5381
5382 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5383 ASSERT_TRUE(prev_rate_settings.has_value());
5384 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5385 kDefaultFramerate);
5386
5387 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5388 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5389 timestamp_ms += 1000 / kDefaultFramerate;
5390 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5391 WaitForEncodedFrame(timestamp_ms);
5392 }
5393 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5394 kDefaultFramerate);
5395 // Capture larger frame to trigger a reconfigure.
5396 codec_height_ *= 2;
5397 codec_width_ *= 2;
5398 timestamp_ms += 1000 / kDefaultFramerate;
5399 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5400 WaitForEncodedFrame(timestamp_ms);
5401
5402 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5403 auto current_rate_settings =
5404 fake_encoder_.GetAndResetLastRateControlSettings();
5405 // Ensure we have actually reconfigured twice
5406 // The rate settings should have been set again even though
5407 // they haven't changed.
5408 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005409 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005410
5411 video_stream_encoder_->Stop();
5412}
5413
philipeld9cc8c02019-09-16 14:53:40 +02005414struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5415 MOCK_METHOD0(RequestEncoderFallback, void());
5416 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 11:30:00 +01005417 MOCK_METHOD1(RequestEncoderSwitch,
5418 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 14:53:40 +02005419};
5420
5421TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5422 constexpr int kDontCare = 100;
5423
5424 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5425 video_send_config_.encoder_settings.encoder_switch_request_callback =
5426 &switch_callback;
5427 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5428 encoder_config.codec_type = kVideoCodecVP8;
5429 webrtc::test::ScopedFieldTrials field_trial(
5430 "WebRTC-NetworkCondition-EncoderSwitch/"
5431 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5432 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5433
5434 // Reset encoder for new configuration to take effect.
5435 ConfigureEncoder(std::move(encoder_config));
5436
5437 // Send one frame to trigger ReconfigureEncoder.
5438 video_source_.IncomingCapturedFrame(
5439 CreateFrame(kDontCare, kDontCare, kDontCare));
5440
5441 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005442 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5443 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005444 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005445 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005446
Henrik Boström381d1092020-05-12 18:49:07 +02005447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005448 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5449 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5450 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005451 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005452 /*rtt_ms=*/0,
5453 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005454
5455 video_stream_encoder_->Stop();
5456}
5457
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005458TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5459 constexpr int kDontCare = 100;
5460
5461 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5462 video_send_config_.encoder_settings.encoder_switch_request_callback =
5463 &switch_callback;
5464 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5465 encoder_config.codec_type = kVideoCodecVP8;
5466 webrtc::test::ScopedFieldTrials field_trial(
5467 "WebRTC-NetworkCondition-EncoderSwitch/"
5468 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5469 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5470
5471 // Reset encoder for new configuration to take effect.
5472 ConfigureEncoder(std::move(encoder_config));
5473
5474 // Send one frame to trigger ReconfigureEncoder.
5475 video_source_.IncomingCapturedFrame(
5476 CreateFrame(kDontCare, kDontCare, kDontCare));
5477
5478 using Config = EncoderSwitchRequestCallback::Config;
5479 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5480 .Times(0);
5481
Henrik Boström381d1092020-05-12 18:49:07 +02005482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005483 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5484 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5485 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5486 /*fraction_lost=*/0,
5487 /*rtt_ms=*/0,
5488 /*cwnd_reduce_ratio=*/0);
5489
5490 video_stream_encoder_->Stop();
5491}
5492
philipeld9cc8c02019-09-16 14:53:40 +02005493TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5494 constexpr int kSufficientBitrateToNotDrop = 1000;
5495 constexpr int kHighRes = 500;
5496 constexpr int kLowRes = 100;
5497
5498 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5499 video_send_config_.encoder_settings.encoder_switch_request_callback =
5500 &switch_callback;
5501 webrtc::test::ScopedFieldTrials field_trial(
5502 "WebRTC-NetworkCondition-EncoderSwitch/"
5503 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5504 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5505 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5506 encoder_config.codec_type = kVideoCodecH264;
5507
5508 // Reset encoder for new configuration to take effect.
5509 ConfigureEncoder(std::move(encoder_config));
5510
5511 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5512 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5513 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005515 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5516 /*stable_target_bitrate=*/
5517 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5518 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005519 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005520 /*rtt_ms=*/0,
5521 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005522
5523 // Send one frame to trigger ReconfigureEncoder.
5524 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5525 WaitForEncodedFrame(1);
5526
5527 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005528 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5529 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005530 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005531 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005532
5533 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5534 WaitForEncodedFrame(2);
5535
5536 video_stream_encoder_->Stop();
5537}
5538
philipel9b058032020-02-10 11:30:00 +01005539TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5540 constexpr int kDontCare = 100;
5541 StrictMock<MockEncoderSelector> encoder_selector;
5542 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5543 &fake_encoder_, &encoder_selector);
5544 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5545
5546 // Reset encoder for new configuration to take effect.
5547 ConfigureEncoder(video_encoder_config_.Copy());
5548
5549 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5550
5551 video_source_.IncomingCapturedFrame(
5552 CreateFrame(kDontCare, kDontCare, kDontCare));
5553 video_stream_encoder_->Stop();
5554
5555 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5556 // to it's factory, so in order for the encoder instance in the
5557 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5558 // reset the |video_stream_encoder_| here.
5559 video_stream_encoder_.reset();
5560}
5561
5562TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5563 constexpr int kDontCare = 100;
5564
5565 NiceMock<MockEncoderSelector> encoder_selector;
5566 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5567 video_send_config_.encoder_settings.encoder_switch_request_callback =
5568 &switch_callback;
5569 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5570 &fake_encoder_, &encoder_selector);
5571 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5572
5573 // Reset encoder for new configuration to take effect.
5574 ConfigureEncoder(video_encoder_config_.Copy());
5575
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005576 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005577 .WillByDefault(Return(SdpVideoFormat("AV1")));
5578 EXPECT_CALL(switch_callback,
5579 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5580 Field(&SdpVideoFormat::name, "AV1"))));
5581
Henrik Boström381d1092020-05-12 18:49:07 +02005582 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005583 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5584 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5585 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005586 /*fraction_lost=*/0,
5587 /*rtt_ms=*/0,
5588 /*cwnd_reduce_ratio=*/0);
5589
5590 video_stream_encoder_->Stop();
5591}
5592
5593TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5594 constexpr int kSufficientBitrateToNotDrop = 1000;
5595 constexpr int kDontCare = 100;
5596
5597 NiceMock<MockVideoEncoder> video_encoder;
5598 NiceMock<MockEncoderSelector> encoder_selector;
5599 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5600 video_send_config_.encoder_settings.encoder_switch_request_callback =
5601 &switch_callback;
5602 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5603 &video_encoder, &encoder_selector);
5604 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5605
5606 // Reset encoder for new configuration to take effect.
5607 ConfigureEncoder(video_encoder_config_.Copy());
5608
5609 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5610 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5611 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005613 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5614 /*stable_target_bitrate=*/
5615 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5616 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005617 /*fraction_lost=*/0,
5618 /*rtt_ms=*/0,
5619 /*cwnd_reduce_ratio=*/0);
5620
5621 ON_CALL(video_encoder, Encode(_, _))
5622 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5623 ON_CALL(encoder_selector, OnEncoderBroken())
5624 .WillByDefault(Return(SdpVideoFormat("AV2")));
5625
5626 rtc::Event encode_attempted;
5627 EXPECT_CALL(switch_callback,
5628 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5629 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5630 EXPECT_EQ(format.name, "AV2");
5631 encode_attempted.Set();
5632 });
5633
5634 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5635 encode_attempted.Wait(3000);
5636
5637 video_stream_encoder_->Stop();
5638
5639 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5640 // to it's factory, so in order for the encoder instance in the
5641 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5642 // reset the |video_stream_encoder_| here.
5643 video_stream_encoder_.reset();
5644}
5645
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005646TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005647 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005648 const int kFrameWidth = 320;
5649 const int kFrameHeight = 180;
5650
5651 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005652 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005654 /*target_bitrate=*/rate,
5655 /*stable_target_bitrate=*/rate,
5656 /*link_allocation=*/rate,
5657 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005658 /*rtt_ms=*/0,
5659 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005660
5661 // Insert a first video frame so that encoder gets configured.
5662 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5663 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5664 frame.set_rotation(kVideoRotation_270);
5665 video_source_.IncomingCapturedFrame(frame);
5666 WaitForEncodedFrame(timestamp_ms);
5667 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5668
5669 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005670 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005672 /*target_bitrate=*/new_stable_rate,
5673 /*stable_target_bitrate=*/new_stable_rate,
5674 /*link_allocation=*/rate,
5675 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005676 /*rtt_ms=*/0,
5677 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005678 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5679 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5680 video_stream_encoder_->Stop();
5681}
5682
5683TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005684 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005685 const int kFrameWidth = 320;
5686 const int kFrameHeight = 180;
5687
5688 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005689 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005691 /*target_bitrate=*/rate,
5692 /*stable_target_bitrate=*/rate,
5693 /*link_allocation=*/rate,
5694 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005695 /*rtt_ms=*/0,
5696 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005697
5698 // Insert a first video frame so that encoder gets configured.
5699 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5700 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5701 frame.set_rotation(kVideoRotation_270);
5702 video_source_.IncomingCapturedFrame(frame);
5703 WaitForEncodedFrame(timestamp_ms);
5704 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5705
5706 // Set a higher target rate without changing the link_allocation. Should not
5707 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005708 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005710 /*target_bitrate=*/rate,
5711 /*stable_target_bitrate=*/new_stable_rate,
5712 /*link_allocation=*/rate,
5713 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005714 /*rtt_ms=*/0,
5715 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005716 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5717 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5718 video_stream_encoder_->Stop();
5719}
5720
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005721TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5722 test::ScopedFieldTrials field_trials(
5723 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5724 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5725 const int kFramerateFps = 30;
5726 const int kWidth = 1920;
5727 const int kHeight = 1080;
5728 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5729 // Works on screenshare mode.
5730 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5731 // We rely on the automatic resolution adaptation, but we handle framerate
5732 // adaptation manually by mocking the stats proxy.
5733 video_source_.set_adaptation_enabled(true);
5734
5735 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005736 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005737 DataRate::BitsPerSec(kTargetBitrateBps),
5738 DataRate::BitsPerSec(kTargetBitrateBps),
5739 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005740 video_stream_encoder_->SetSource(&video_source_,
5741 webrtc::DegradationPreference::BALANCED);
5742 VerifyNoLimitation(video_source_.sink_wants());
5743
5744 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5745 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5746
5747 // Pass enough frames with the full update to trigger animation detection.
5748 for (int i = 0; i < kNumFrames; ++i) {
5749 int64_t timestamp_ms =
5750 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5751 frame.set_ntp_time_ms(timestamp_ms);
5752 frame.set_timestamp_us(timestamp_ms * 1000);
5753 video_source_.IncomingCapturedFrame(frame);
5754 WaitForEncodedFrame(timestamp_ms);
5755 }
5756
5757 // Resolution should be limited.
5758 rtc::VideoSinkWants expected;
5759 expected.max_framerate_fps = kFramerateFps;
5760 expected.max_pixel_count = 1280 * 720 + 1;
5761 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5762
5763 // Pass one frame with no known update.
5764 // Resolution cap should be removed immediately.
5765 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5766 frame.set_ntp_time_ms(timestamp_ms);
5767 frame.set_timestamp_us(timestamp_ms * 1000);
5768 frame.clear_update_rect();
5769
5770 video_source_.IncomingCapturedFrame(frame);
5771 WaitForEncodedFrame(timestamp_ms);
5772
5773 // Resolution should be unlimited now.
5774 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5775
5776 video_stream_encoder_->Stop();
5777}
5778
perkj26091b12016-09-01 01:17:40 -07005779} // namespace webrtc