blob: cf9e790fcffe16d8c431977213c8d048430028af [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020040#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "system_wrappers/include/sleep.h"
42#include "test/encoder_settings.h"
43#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020044#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010045#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/gmock.h"
47#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020048#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020049#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070050
51namespace webrtc {
52
sprang57c2fff2017-01-16 06:24:02 -080053using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020054using ::testing::AllOf;
55using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010056using ::testing::Matcher;
57using ::testing::NiceMock;
58using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080060
perkj803d97f2016-11-01 11:45:46 -070061namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020062const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010063const int kQpLow = 1;
64const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int kMinFramerateFps = 2;
66const int kMinBalancedFramerateFps = 7;
67const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080068const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010069const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020070const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010071const uint32_t kSimulcastTargetBitrateBps = 3150000;
72const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080073const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070074const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020076const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020077const VideoEncoder::ResolutionBitrateLimits
78 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
79const VideoEncoder::ResolutionBitrateLimits
80 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080081
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020082uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
83 0x00, 0x00, 0x03, 0x03, 0xF4,
84 0x05, 0x03, 0xC7, 0xE0, 0x1B,
85 0x41, 0x10, 0x8D, 0x00};
86
perkj803d97f2016-11-01 11:45:46 -070087class TestBuffer : public webrtc::I420Buffer {
88 public:
89 TestBuffer(rtc::Event* event, int width, int height)
90 : I420Buffer(width, height), event_(event) {}
91
92 private:
93 friend class rtc::RefCountedObject<TestBuffer>;
94 ~TestBuffer() override {
95 if (event_)
96 event_->Set();
97 }
98 rtc::Event* const event_;
99};
100
Noah Richards51db4212019-06-12 06:59:12 -0700101// A fake native buffer that can't be converted to I420.
102class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
103 public:
104 FakeNativeBuffer(rtc::Event* event, int width, int height)
105 : event_(event), width_(width), height_(height) {}
106 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
107 int width() const override { return width_; }
108 int height() const override { return height_; }
109 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
110 return nullptr;
111 }
112
113 private:
114 friend class rtc::RefCountedObject<FakeNativeBuffer>;
115 ~FakeNativeBuffer() override {
116 if (event_)
117 event_->Set();
118 }
119 rtc::Event* const event_;
120 const int width_;
121 const int height_;
122};
123
Niels Möller7dc26b72017-12-06 10:27:48 +0100124class CpuOveruseDetectorProxy : public OveruseFrameDetector {
125 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200126 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
127 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100128 last_target_framerate_fps_(-1) {}
129 virtual ~CpuOveruseDetectorProxy() {}
130
131 void OnTargetFramerateUpdated(int framerate_fps) override {
132 rtc::CritScope cs(&lock_);
133 last_target_framerate_fps_ = framerate_fps;
134 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
135 }
136
137 int GetLastTargetFramerate() {
138 rtc::CritScope cs(&lock_);
139 return last_target_framerate_fps_;
140 }
141
Niels Möller4db138e2018-04-19 09:04:13 +0200142 CpuOveruseOptions GetOptions() { return options_; }
143
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 private:
145 rtc::CriticalSection lock_;
146 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
147};
148
Henrik Boström91aa7322020-04-28 12:24:33 +0200149class FakeQualityScalerQpUsageHandlerCallback
150 : public QualityScalerQpUsageHandlerCallbackInterface {
151 public:
152 FakeQualityScalerQpUsageHandlerCallback()
153 : QualityScalerQpUsageHandlerCallbackInterface() {}
154 ~FakeQualityScalerQpUsageHandlerCallback() override {}
155
156 void OnQpUsageHandled(bool clear_qp_samples) override {
157 clear_qp_samples_result_ = clear_qp_samples;
158 }
159
160 absl::optional<bool> clear_qp_samples_result() const {
161 return clear_qp_samples_result_;
162 }
163
164 private:
165 absl::optional<bool> clear_qp_samples_result_;
166};
167
mflodmancc3d4422017-08-03 08:27:51 -0700168class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700169 public:
Niels Möller213618e2018-07-24 09:29:58 +0200170 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200171 const VideoStreamEncoderSettings& settings,
172 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100173 : VideoStreamEncoder(Clock::GetRealTimeClock(),
174 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200175 stats_proxy,
176 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200177 std::unique_ptr<OveruseFrameDetector>(
178 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100179 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100180 task_queue_factory),
Henrik Boströmdc4f75f2020-04-20 12:04:12 +0200181 fake_cpu_resource_(std::make_unique<FakeResource>("FakeResource[CPU]")),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100182 fake_quality_resource_(
Henrik Boströmdc4f75f2020-04-20 12:04:12 +0200183 std::make_unique<FakeResource>("FakeResource[QP]")) {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200184 InjectAdaptationResource(fake_quality_resource_.get(),
185 VideoAdaptationReason::kQuality);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100186 InjectAdaptationResource(fake_cpu_resource_.get(),
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200187 VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100188 }
perkj803d97f2016-11-01 11:45:46 -0700189
kthelgason2fc52542017-03-03 00:24:41 -0800190 // This is used as a synchronisation mechanism, to make sure that the
191 // encoder queue is not blocked before we start sending it frames.
192 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100193 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200194 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800195 ASSERT_TRUE(event.Wait(5000));
196 }
197
Henrik Boström91aa7322020-04-28 12:24:33 +0200198 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200199 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200200 rtc::Event event;
201 encoder_queue()->PostTask([this, &event] {
202 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
203 event.Set();
204 });
205 ASSERT_TRUE(event.Wait(5000));
206 }
207 void TriggerCpuUnderuse() {
208 rtc::Event event;
209 encoder_queue()->PostTask([this, &event] {
210 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
211 event.Set();
212 });
213 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200214 }
kthelgason876222f2016-11-29 01:44:11 -0800215
Henrik Boström91aa7322020-04-28 12:24:33 +0200216 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200217 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200218 rtc::Event event;
219 encoder_queue()->PostTask([this, &event] {
220 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
221 event.Set();
222 });
223 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200224 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200225 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200226 rtc::Event event;
227 encoder_queue()->PostTask([this, &event] {
228 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
229 event.Set();
230 });
231 ASSERT_TRUE(event.Wait(5000));
232 }
233
234 // Fakes high QP resource usage measurements on the real
235 // QualityScalerResource. Returns whether or not QP samples would have been
236 // cleared if this had been a real signal from the QualityScaler.
237 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
238 rtc::Event event;
239 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
240 new FakeQualityScalerQpUsageHandlerCallback();
241 encoder_queue()->PostTask([this, &event, callback] {
242 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
243 event.Set();
244 });
245 EXPECT_TRUE(event.Wait(5000));
246 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
247 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200248 }
sprangfda496a2017-06-15 04:21:07 -0700249
Niels Möller7dc26b72017-12-06 10:27:48 +0100250 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100251 std::unique_ptr<FakeResource> fake_cpu_resource_;
252 std::unique_ptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700253};
254
asapersson5f7226f2016-11-25 04:37:00 -0800255class VideoStreamFactory
256 : public VideoEncoderConfig::VideoStreamFactoryInterface {
257 public:
sprangfda496a2017-06-15 04:21:07 -0700258 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
259 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800260 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700261 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800262 }
263
264 private:
265 std::vector<VideoStream> CreateEncoderStreams(
266 int width,
267 int height,
268 const VideoEncoderConfig& encoder_config) override {
269 std::vector<VideoStream> streams =
270 test::CreateVideoStreams(width, height, encoder_config);
271 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100272 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700273 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800274 }
275 return streams;
276 }
sprangfda496a2017-06-15 04:21:07 -0700277
asapersson5f7226f2016-11-25 04:37:00 -0800278 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700279 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800280};
281
Noah Richards51db4212019-06-12 06:59:12 -0700282// Simulates simulcast behavior and makes highest stream resolutions divisible
283// by 4.
284class CroppingVideoStreamFactory
285 : public VideoEncoderConfig::VideoStreamFactoryInterface {
286 public:
287 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
288 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
289 EXPECT_GT(num_temporal_layers, 0u);
290 EXPECT_GT(framerate, 0);
291 }
292
293 private:
294 std::vector<VideoStream> CreateEncoderStreams(
295 int width,
296 int height,
297 const VideoEncoderConfig& encoder_config) override {
298 std::vector<VideoStream> streams = test::CreateVideoStreams(
299 width - width % 4, height - height % 4, encoder_config);
300 for (VideoStream& stream : streams) {
301 stream.num_temporal_layers = num_temporal_layers_;
302 stream.max_framerate = framerate_;
303 }
304 return streams;
305 }
306
307 const size_t num_temporal_layers_;
308 const int framerate_;
309};
310
sprangb1ca0732017-02-01 08:38:12 -0800311class AdaptingFrameForwarder : public test::FrameForwarder {
312 public:
313 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700314 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800315
316 void set_adaptation_enabled(bool enabled) {
317 rtc::CritScope cs(&crit_);
318 adaptation_enabled_ = enabled;
319 }
320
asaperssonfab67072017-04-04 05:51:49 -0700321 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800322 rtc::CritScope cs(&crit_);
323 return adaptation_enabled_;
324 }
325
asapersson09f05612017-05-15 23:40:18 -0700326 rtc::VideoSinkWants last_wants() const {
327 rtc::CritScope cs(&crit_);
328 return last_wants_;
329 }
330
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200331 absl::optional<int> last_sent_width() const { return last_width_; }
332 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800333
sprangb1ca0732017-02-01 08:38:12 -0800334 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
335 int cropped_width = 0;
336 int cropped_height = 0;
337 int out_width = 0;
338 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700339 if (adaption_enabled()) {
340 if (adapter_.AdaptFrameResolution(
341 video_frame.width(), video_frame.height(),
342 video_frame.timestamp_us() * 1000, &cropped_width,
343 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100344 VideoFrame adapted_frame =
345 VideoFrame::Builder()
346 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
347 nullptr, out_width, out_height))
348 .set_timestamp_rtp(99)
349 .set_timestamp_ms(99)
350 .set_rotation(kVideoRotation_0)
351 .build();
sprangc5d62e22017-04-02 23:53:04 -0700352 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100353 if (video_frame.has_update_rect()) {
354 adapted_frame.set_update_rect(
355 video_frame.update_rect().ScaleWithFrame(
356 video_frame.width(), video_frame.height(), 0, 0,
357 video_frame.width(), video_frame.height(), out_width,
358 out_height));
359 }
sprangc5d62e22017-04-02 23:53:04 -0700360 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800361 last_width_.emplace(adapted_frame.width());
362 last_height_.emplace(adapted_frame.height());
363 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200364 last_width_ = absl::nullopt;
365 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700366 }
sprangb1ca0732017-02-01 08:38:12 -0800367 } else {
368 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800369 last_width_.emplace(video_frame.width());
370 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800371 }
372 }
373
374 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
375 const rtc::VideoSinkWants& wants) override {
376 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700377 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100378 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800379 test::FrameForwarder::AddOrUpdateSink(sink, wants);
380 }
sprangb1ca0732017-02-01 08:38:12 -0800381 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700382 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
383 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200384 absl::optional<int> last_width_;
385 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800386};
sprangc5d62e22017-04-02 23:53:04 -0700387
Niels Möller213618e2018-07-24 09:29:58 +0200388// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700389class MockableSendStatisticsProxy : public SendStatisticsProxy {
390 public:
391 MockableSendStatisticsProxy(Clock* clock,
392 const VideoSendStream::Config& config,
393 VideoEncoderConfig::ContentType content_type)
394 : SendStatisticsProxy(clock, config, content_type) {}
395
396 VideoSendStream::Stats GetStats() override {
397 rtc::CritScope cs(&lock_);
398 if (mock_stats_)
399 return *mock_stats_;
400 return SendStatisticsProxy::GetStats();
401 }
402
Niels Möller213618e2018-07-24 09:29:58 +0200403 int GetInputFrameRate() const override {
404 rtc::CritScope cs(&lock_);
405 if (mock_stats_)
406 return mock_stats_->input_frame_rate;
407 return SendStatisticsProxy::GetInputFrameRate();
408 }
sprangc5d62e22017-04-02 23:53:04 -0700409 void SetMockStats(const VideoSendStream::Stats& stats) {
410 rtc::CritScope cs(&lock_);
411 mock_stats_.emplace(stats);
412 }
413
414 void ResetMockStats() {
415 rtc::CritScope cs(&lock_);
416 mock_stats_.reset();
417 }
418
419 private:
420 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200421 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700422};
423
sprang4847ae62017-06-27 07:06:52 -0700424class MockBitrateObserver : public VideoBitrateAllocationObserver {
425 public:
Erik Språng566124a2018-04-23 12:32:22 +0200426 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700427};
428
philipel9b058032020-02-10 11:30:00 +0100429class MockEncoderSelector
430 : public VideoEncoderFactory::EncoderSelectorInterface {
431 public:
432 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
Mirta Dvornicic4f34d782020-02-26 13:01:19 +0100433 MOCK_METHOD1(OnAvailableBitrate,
philipel9b058032020-02-10 11:30:00 +0100434 absl::optional<SdpVideoFormat>(const DataRate& rate));
435 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
436};
437
perkj803d97f2016-11-01 11:45:46 -0700438} // namespace
439
mflodmancc3d4422017-08-03 08:27:51 -0700440class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700441 public:
442 static const int kDefaultTimeoutMs = 30 * 1000;
443
mflodmancc3d4422017-08-03 08:27:51 -0700444 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700445 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700446 codec_width_(320),
447 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200448 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200449 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700450 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200451 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700452 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700453 Clock::GetRealTimeClock(),
454 video_send_config_,
455 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700456 sink_(&fake_encoder_) {}
457
458 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700459 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700460 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200461 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800462 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200463 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200464 video_send_config_.rtp.payload_name = "FAKE";
465 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700466
Per512ecb32016-09-23 15:52:06 +0200467 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200468 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700469 video_encoder_config.video_stream_factory =
470 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100471 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700472
473 // Framerate limit is specified by the VideoStreamFactory.
474 std::vector<VideoStream> streams =
475 video_encoder_config.video_stream_factory->CreateEncoderStreams(
476 codec_width_, codec_height_, video_encoder_config);
477 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100478 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700479
Niels Möllerf1338562018-04-26 09:51:47 +0200480 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800481 }
482
Niels Möllerf1338562018-04-26 09:51:47 +0200483 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700484 if (video_stream_encoder_)
485 video_stream_encoder_->Stop();
486 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200487 stats_proxy_.get(), video_send_config_.encoder_settings,
488 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700489 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
490 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700491 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700492 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
493 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200494 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700495 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800496 }
497
498 void ResetEncoder(const std::string& payload_name,
499 size_t num_streams,
500 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700501 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700502 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200503 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800504
505 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200506 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800507 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100508 video_encoder_config.max_bitrate_bps =
509 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800510 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700511 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
512 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700513 video_encoder_config.content_type =
514 screenshare ? VideoEncoderConfig::ContentType::kScreen
515 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700516 if (payload_name == "VP9") {
517 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
518 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
519 video_encoder_config.encoder_specific_settings =
520 new rtc::RefCountedObject<
521 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
522 }
Niels Möllerf1338562018-04-26 09:51:47 +0200523 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700524 }
525
sprang57c2fff2017-01-16 06:24:02 -0800526 VideoFrame CreateFrame(int64_t ntp_time_ms,
527 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100528 VideoFrame frame =
529 VideoFrame::Builder()
530 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
531 destruction_event, codec_width_, codec_height_))
532 .set_timestamp_rtp(99)
533 .set_timestamp_ms(99)
534 .set_rotation(kVideoRotation_0)
535 .build();
sprang57c2fff2017-01-16 06:24:02 -0800536 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700537 return frame;
538 }
539
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100540 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
541 rtc::Event* destruction_event,
542 int offset_x) const {
543 VideoFrame frame =
544 VideoFrame::Builder()
545 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
546 destruction_event, codec_width_, codec_height_))
547 .set_timestamp_rtp(99)
548 .set_timestamp_ms(99)
549 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100550 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100551 .build();
552 frame.set_ntp_time_ms(ntp_time_ms);
553 return frame;
554 }
555
sprang57c2fff2017-01-16 06:24:02 -0800556 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100557 VideoFrame frame =
558 VideoFrame::Builder()
559 .set_video_frame_buffer(
560 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
561 .set_timestamp_rtp(99)
562 .set_timestamp_ms(99)
563 .set_rotation(kVideoRotation_0)
564 .build();
sprang57c2fff2017-01-16 06:24:02 -0800565 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700566 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700567 return frame;
568 }
569
Noah Richards51db4212019-06-12 06:59:12 -0700570 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
571 rtc::Event* destruction_event,
572 int width,
573 int height) const {
574 VideoFrame frame =
575 VideoFrame::Builder()
576 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
577 destruction_event, width, height))
578 .set_timestamp_rtp(99)
579 .set_timestamp_ms(99)
580 .set_rotation(kVideoRotation_0)
581 .build();
582 frame.set_ntp_time_ms(ntp_time_ms);
583 return frame;
584 }
585
586 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
587 rtc::Event* destruction_event) const {
588 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
589 codec_height_);
590 }
591
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100592 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
593 MockBitrateObserver bitrate_observer;
594 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
595
596 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
597 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200598 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100599 DataRate::BitsPerSec(kTargetBitrateBps),
600 DataRate::BitsPerSec(kTargetBitrateBps),
601 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100602
603 video_source_.IncomingCapturedFrame(
604 CreateFrame(1, codec_width_, codec_height_));
605 WaitForEncodedFrame(1);
606 }
607
asapersson02465b82017-04-10 01:12:52 -0700608 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700609 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700610 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
611 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700612 }
613
asapersson09f05612017-05-15 23:40:18 -0700614 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
615 const rtc::VideoSinkWants& wants2) {
616 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
617 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
618 }
619
Åsa Persson8c1bf952018-09-13 10:42:19 +0200620 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
621 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
622 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
623 EXPECT_FALSE(wants.target_pixel_count);
624 }
625
asapersson09f05612017-05-15 23:40:18 -0700626 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
627 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200628 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700629 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
630 EXPECT_GT(wants1.max_pixel_count, 0);
631 }
632
633 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
634 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200635 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700636 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
637 }
638
asaperssonf7e294d2017-06-13 23:25:22 -0700639 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
640 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200641 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700642 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
643 }
644
645 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
646 const rtc::VideoSinkWants& wants2) {
647 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
648 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
649 }
650
651 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
652 const rtc::VideoSinkWants& wants2) {
653 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
654 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
655 }
656
657 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
658 const rtc::VideoSinkWants& wants2) {
659 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
660 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
661 EXPECT_GT(wants1.max_pixel_count, 0);
662 }
663
664 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
665 const rtc::VideoSinkWants& wants2) {
666 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
667 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
668 }
669
asapersson09f05612017-05-15 23:40:18 -0700670 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
671 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200672 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700673 EXPECT_LT(wants.max_pixel_count, pixel_count);
674 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700675 }
676
677 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
678 EXPECT_LT(wants.max_framerate_fps, fps);
679 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
680 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700681 }
682
asaperssonf7e294d2017-06-13 23:25:22 -0700683 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
684 int expected_fps) {
685 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
686 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
687 EXPECT_FALSE(wants.target_pixel_count);
688 }
689
Jonathan Yubc771b72017-12-08 17:04:29 -0800690 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
691 int last_frame_pixels) {
692 // Balanced mode should always scale FPS to the desired range before
693 // attempting to scale resolution.
694 int fps_limit = wants.max_framerate_fps;
695 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100696 EXPECT_LE(7, fps_limit);
697 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800698 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100699 EXPECT_LE(10, fps_limit);
700 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800701 } else if (last_frame_pixels <= 640 * 480) {
702 EXPECT_LE(15, fps_limit);
703 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200704 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800705 }
706 }
707
sprang4847ae62017-06-27 07:06:52 -0700708 void WaitForEncodedFrame(int64_t expected_ntp_time) {
709 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100710 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700711 }
712
713 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
714 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100715 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700716 return ok;
717 }
718
719 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
720 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100721 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700722 }
723
724 void ExpectDroppedFrame() {
725 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100726 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700727 }
728
729 bool WaitForFrame(int64_t timeout_ms) {
730 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100731 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700732 return ok;
733 }
734
perkj26091b12016-09-01 01:17:40 -0700735 class TestEncoder : public test::FakeEncoder {
736 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100737 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700738
asaperssonfab67072017-04-04 05:51:49 -0700739 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800740 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700741 return config_;
742 }
743
744 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800745 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700746 block_next_encode_ = true;
747 }
748
Erik Språngaed30702018-11-05 12:57:17 +0100749 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800750 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100751 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100752 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100753 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100754 info.scaling_settings = VideoEncoder::ScalingSettings(
755 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100756 }
757 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100758 for (int i = 0; i < kMaxSpatialLayers; ++i) {
759 if (temporal_layers_supported_[i]) {
760 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
761 info.fps_allocation[i].resize(num_layers);
762 }
763 }
Erik Språngaed30702018-11-05 12:57:17 +0100764 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200765
766 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100767 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100768 return info;
kthelgason876222f2016-11-29 01:44:11 -0800769 }
770
Erik Språngb7cb7b52019-02-26 15:52:33 +0100771 int32_t RegisterEncodeCompleteCallback(
772 EncodedImageCallback* callback) override {
773 rtc::CritScope lock(&local_crit_sect_);
774 encoded_image_callback_ = callback;
775 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
776 }
777
perkjfa10b552016-10-02 23:45:26 -0700778 void ContinueEncode() { continue_encode_event_.Set(); }
779
780 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
781 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800782 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700783 EXPECT_EQ(timestamp_, timestamp);
784 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
785 }
786
kthelgason2fc52542017-03-03 00:24:41 -0800787 void SetQualityScaling(bool b) {
788 rtc::CritScope lock(&local_crit_sect_);
789 quality_scaling_ = b;
790 }
kthelgasonad9010c2017-02-14 00:46:51 -0800791
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100792 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
793 rtc::CritScope lock(&local_crit_sect_);
794 requested_resolution_alignment_ = requested_resolution_alignment;
795 }
796
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100797 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
798 rtc::CritScope lock(&local_crit_sect_);
799 is_hardware_accelerated_ = is_hardware_accelerated;
800 }
801
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100802 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
803 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
804 rtc::CritScope lock(&local_crit_sect_);
805 temporal_layers_supported_[spatial_idx] = supported;
806 }
807
Sergey Silkin6456e352019-07-08 17:56:40 +0200808 void SetResolutionBitrateLimits(
809 std::vector<ResolutionBitrateLimits> thresholds) {
810 rtc::CritScope cs(&local_crit_sect_);
811 resolution_bitrate_limits_ = thresholds;
812 }
813
sprangfe627f32017-03-29 08:24:59 -0700814 void ForceInitEncodeFailure(bool force_failure) {
815 rtc::CritScope lock(&local_crit_sect_);
816 force_init_encode_failed_ = force_failure;
817 }
818
Niels Möller6bb5ab92019-01-11 11:11:10 +0100819 void SimulateOvershoot(double rate_factor) {
820 rtc::CritScope lock(&local_crit_sect_);
821 rate_factor_ = rate_factor;
822 }
823
Erik Språngd7329ca2019-02-21 21:19:53 +0100824 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100825 rtc::CritScope lock(&local_crit_sect_);
826 return last_framerate_;
827 }
828
Erik Språngd7329ca2019-02-21 21:19:53 +0100829 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100830 rtc::CritScope lock(&local_crit_sect_);
831 return last_update_rect_;
832 }
833
Niels Möller87e2d782019-03-07 10:18:23 +0100834 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100835 rtc::CritScope lock(&local_crit_sect_);
836 return last_frame_types_;
837 }
838
839 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100840 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100841 keyframe ? VideoFrameType::kVideoFrameKey
842 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100843 {
844 rtc::CritScope lock(&local_crit_sect_);
845 last_frame_types_ = frame_type;
846 }
Niels Möllerb859b322019-03-07 12:40:01 +0100847 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100848 }
849
Erik Språngb7cb7b52019-02-26 15:52:33 +0100850 void InjectEncodedImage(const EncodedImage& image) {
851 rtc::CritScope lock(&local_crit_sect_);
852 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
853 }
854
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200855 void InjectEncodedImage(const EncodedImage& image,
856 const CodecSpecificInfo* codec_specific_info,
857 const RTPFragmentationHeader* fragmentation) {
858 rtc::CritScope lock(&local_crit_sect_);
859 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
860 fragmentation);
861 }
862
Erik Språngd7329ca2019-02-21 21:19:53 +0100863 void ExpectNullFrame() {
864 rtc::CritScope lock(&local_crit_sect_);
865 expect_null_frame_ = true;
866 }
867
Erik Språng5056af02019-09-02 15:53:11 +0200868 absl::optional<VideoEncoder::RateControlParameters>
869 GetAndResetLastRateControlSettings() {
870 auto settings = last_rate_control_settings_;
871 last_rate_control_settings_.reset();
872 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100873 }
874
Sergey Silkin5ee69672019-07-02 14:18:34 +0200875 int GetNumEncoderInitializations() const {
876 rtc::CritScope lock(&local_crit_sect_);
877 return num_encoder_initializations_;
878 }
879
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200880 int GetNumSetRates() const {
881 rtc::CritScope lock(&local_crit_sect_);
882 return num_set_rates_;
883 }
884
perkjfa10b552016-10-02 23:45:26 -0700885 private:
perkj26091b12016-09-01 01:17:40 -0700886 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100887 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700888 bool block_encode;
889 {
brandtre78d2662017-01-16 05:57:16 -0800890 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100891 if (expect_null_frame_) {
892 EXPECT_EQ(input_image.timestamp(), 0u);
893 EXPECT_EQ(input_image.width(), 1);
894 last_frame_types_ = *frame_types;
895 expect_null_frame_ = false;
896 } else {
897 EXPECT_GT(input_image.timestamp(), timestamp_);
898 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
899 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
900 }
perkj26091b12016-09-01 01:17:40 -0700901
902 timestamp_ = input_image.timestamp();
903 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700904 last_input_width_ = input_image.width();
905 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700906 block_encode = block_next_encode_;
907 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100908 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100909 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700910 }
Niels Möllerb859b322019-03-07 12:40:01 +0100911 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700912 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700913 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700914 return result;
915 }
916
sprangfe627f32017-03-29 08:24:59 -0700917 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200918 const Settings& settings) override {
919 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200920
sprangfe627f32017-03-29 08:24:59 -0700921 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100922 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200923
924 ++num_encoder_initializations_;
925
Erik Språng82fad3d2018-03-21 09:57:23 +0100926 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700927 // Simulate setting up temporal layers, in order to validate the life
928 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100929 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200930 frame_buffer_controller_ =
931 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700932 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100933 if (force_init_encode_failed_) {
934 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700935 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100936 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100937
Erik Språngb7cb7b52019-02-26 15:52:33 +0100938 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700939 return res;
940 }
941
Erik Språngb7cb7b52019-02-26 15:52:33 +0100942 int32_t Release() override {
943 rtc::CritScope lock(&local_crit_sect_);
944 EXPECT_NE(initialized_, EncoderState::kUninitialized);
945 initialized_ = EncoderState::kUninitialized;
946 return FakeEncoder::Release();
947 }
948
Erik Språng16cb8f52019-04-12 13:59:09 +0200949 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100950 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200951 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100952 VideoBitrateAllocation adjusted_rate_allocation;
953 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
954 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200955 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100956 adjusted_rate_allocation.SetBitrate(
957 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200958 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100959 rate_factor_));
960 }
961 }
962 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200963 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200964 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200965 RateControlParameters adjusted_paramters = parameters;
966 adjusted_paramters.bitrate = adjusted_rate_allocation;
967 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100968 }
969
brandtre78d2662017-01-16 05:57:16 -0800970 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100971 enum class EncoderState {
972 kUninitialized,
973 kInitializationFailed,
974 kInitialized
975 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
976 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700977 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700978 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700979 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
980 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
981 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
982 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
983 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100984 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100985 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100986 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700987 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100988 absl::optional<bool>
989 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
990 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700991 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100992 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
993 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200994 absl::optional<VideoEncoder::RateControlParameters>
995 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100996 VideoFrame::UpdateRect last_update_rect_
997 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100998 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100999 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001000 EncodedImageCallback* encoded_image_callback_
1001 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001002 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001003 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001004 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1005 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001006 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001007 };
1008
mflodmancc3d4422017-08-03 08:27:51 -07001009 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001010 public:
1011 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001012 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001013
perkj26091b12016-09-01 01:17:40 -07001014 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001015 EXPECT_TRUE(
1016 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1017 }
1018
1019 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1020 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001021 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001022 if (!encoded_frame_event_.Wait(timeout_ms))
1023 return false;
perkj26091b12016-09-01 01:17:40 -07001024 {
1025 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001026 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001027 }
1028 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001029 return true;
perkj26091b12016-09-01 01:17:40 -07001030 }
1031
sprangb1ca0732017-02-01 08:38:12 -08001032 void WaitForEncodedFrame(uint32_t expected_width,
1033 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001034 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001035 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001036 }
1037
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001038 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001039 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001040 uint32_t width = 0;
1041 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001042 {
1043 rtc::CritScope lock(&crit_);
1044 width = last_width_;
1045 height = last_height_;
1046 }
1047 EXPECT_EQ(expected_height, height);
1048 EXPECT_EQ(expected_width, width);
1049 }
1050
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001051 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1052 int width = 0;
1053 int height = 0;
1054 {
1055 rtc::CritScope lock(&crit_);
1056 width = last_width_;
1057 height = last_height_;
1058 }
1059 EXPECT_EQ(width % resolution_alignment, 0);
1060 EXPECT_EQ(height % resolution_alignment, 0);
1061 }
1062
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001063 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1064 VideoRotation rotation;
1065 {
1066 rtc::CritScope lock(&crit_);
1067 rotation = last_rotation_;
1068 }
1069 EXPECT_EQ(expected_rotation, rotation);
1070 }
1071
kthelgason2fc52542017-03-03 00:24:41 -08001072 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001073
sprangc5d62e22017-04-02 23:53:04 -07001074 bool WaitForFrame(int64_t timeout_ms) {
1075 return encoded_frame_event_.Wait(timeout_ms);
1076 }
1077
perkj26091b12016-09-01 01:17:40 -07001078 void SetExpectNoFrames() {
1079 rtc::CritScope lock(&crit_);
1080 expect_frames_ = false;
1081 }
1082
asaperssonfab67072017-04-04 05:51:49 -07001083 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001084 rtc::CritScope lock(&crit_);
1085 return number_of_reconfigurations_;
1086 }
1087
asaperssonfab67072017-04-04 05:51:49 -07001088 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001089 rtc::CritScope lock(&crit_);
1090 return min_transmit_bitrate_bps_;
1091 }
1092
Erik Språngd7329ca2019-02-21 21:19:53 +01001093 void SetNumExpectedLayers(size_t num_layers) {
1094 rtc::CritScope lock(&crit_);
1095 num_expected_layers_ = num_layers;
1096 }
1097
Erik Språngb7cb7b52019-02-26 15:52:33 +01001098 int64_t GetLastCaptureTimeMs() const {
1099 rtc::CritScope lock(&crit_);
1100 return last_capture_time_ms_;
1101 }
1102
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001103 std::vector<uint8_t> GetLastEncodedImageData() {
1104 rtc::CritScope lock(&crit_);
1105 return std::move(last_encoded_image_data_);
1106 }
1107
1108 RTPFragmentationHeader GetLastFragmentation() {
1109 rtc::CritScope lock(&crit_);
1110 return std::move(last_fragmentation_);
1111 }
1112
perkj26091b12016-09-01 01:17:40 -07001113 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001114 Result OnEncodedImage(
1115 const EncodedImage& encoded_image,
1116 const CodecSpecificInfo* codec_specific_info,
1117 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001118 rtc::CritScope lock(&crit_);
1119 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001120 last_encoded_image_data_ = std::vector<uint8_t>(
1121 encoded_image.data(), encoded_image.data() + encoded_image.size());
1122 if (fragmentation) {
1123 last_fragmentation_.CopyFrom(*fragmentation);
1124 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001125 uint32_t timestamp = encoded_image.Timestamp();
1126 if (last_timestamp_ != timestamp) {
1127 num_received_layers_ = 1;
1128 } else {
1129 ++num_received_layers_;
1130 }
1131 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001132 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001133 last_width_ = encoded_image._encodedWidth;
1134 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001135 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001136 if (num_received_layers_ == num_expected_layers_) {
1137 encoded_frame_event_.Set();
1138 }
sprangb1ca0732017-02-01 08:38:12 -08001139 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001140 }
1141
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001142 void OnEncoderConfigurationChanged(
1143 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001144 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001145 VideoEncoderConfig::ContentType content_type,
1146 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001147 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001148 ++number_of_reconfigurations_;
1149 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1150 }
1151
perkj26091b12016-09-01 01:17:40 -07001152 rtc::CriticalSection crit_;
1153 TestEncoder* test_encoder_;
1154 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001155 std::vector<uint8_t> last_encoded_image_data_;
1156 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001157 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001158 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001159 uint32_t last_height_ = 0;
1160 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001161 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001162 size_t num_expected_layers_ = 1;
1163 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001164 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001165 int number_of_reconfigurations_ = 0;
1166 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001167 };
1168
Sergey Silkin5ee69672019-07-02 14:18:34 +02001169 class VideoBitrateAllocatorProxyFactory
1170 : public VideoBitrateAllocatorFactory {
1171 public:
1172 VideoBitrateAllocatorProxyFactory()
1173 : bitrate_allocator_factory_(
1174 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1175
1176 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1177 const VideoCodec& codec) override {
1178 rtc::CritScope lock(&crit_);
1179 codec_config_ = codec;
1180 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1181 }
1182
1183 VideoCodec codec_config() const {
1184 rtc::CritScope lock(&crit_);
1185 return codec_config_;
1186 }
1187
1188 private:
1189 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1190
1191 rtc::CriticalSection crit_;
1192 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1193 };
1194
perkj26091b12016-09-01 01:17:40 -07001195 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001196 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001197 int codec_width_;
1198 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001199 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001200 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001201 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001202 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001203 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001204 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001205 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001206 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001207 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001208 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001209};
1210
mflodmancc3d4422017-08-03 08:27:51 -07001211TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001212 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001213 DataRate::BitsPerSec(kTargetBitrateBps),
1214 DataRate::BitsPerSec(kTargetBitrateBps),
1215 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001216 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001217 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001218 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001219 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001220 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001221}
1222
mflodmancc3d4422017-08-03 08:27:51 -07001223TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001224 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001225 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001226 // The encoder will cache up to one frame for a short duration. Adding two
1227 // frames means that the first frame will be dropped and the second frame will
1228 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001229 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001230 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001231 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001232
Erik Språng4c6ca302019-04-08 15:14:01 +02001233 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001234 DataRate::BitsPerSec(kTargetBitrateBps),
1235 DataRate::BitsPerSec(kTargetBitrateBps),
1236 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001237
Sebastian Janssona3177052018-04-10 13:05:49 +02001238 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001239 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001240 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1241
1242 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001243 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001244}
1245
mflodmancc3d4422017-08-03 08:27:51 -07001246TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001247 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001248 DataRate::BitsPerSec(kTargetBitrateBps),
1249 DataRate::BitsPerSec(kTargetBitrateBps),
1250 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001251 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001252 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001253
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001254 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
1255 DataRate::BitsPerSec(0),
1256 DataRate::BitsPerSec(0), 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001257 // The encoder will cache up to one frame for a short duration. Adding two
1258 // frames means that the first frame will be dropped and the second frame will
1259 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001260 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001261 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001262
Erik Språng4c6ca302019-04-08 15:14:01 +02001263 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001264 DataRate::BitsPerSec(kTargetBitrateBps),
1265 DataRate::BitsPerSec(kTargetBitrateBps),
1266 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001267 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001268 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1269 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001270 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001271}
1272
mflodmancc3d4422017-08-03 08:27:51 -07001273TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001274 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001275 DataRate::BitsPerSec(kTargetBitrateBps),
1276 DataRate::BitsPerSec(kTargetBitrateBps),
1277 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001278 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001279 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001280
1281 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001282 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001283
perkja49cbd32016-09-16 07:53:41 -07001284 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001285 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001286 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001287}
1288
mflodmancc3d4422017-08-03 08:27:51 -07001289TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001290 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001291 DataRate::BitsPerSec(kTargetBitrateBps),
1292 DataRate::BitsPerSec(kTargetBitrateBps),
1293 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001294
perkja49cbd32016-09-16 07:53:41 -07001295 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001296 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001297
mflodmancc3d4422017-08-03 08:27:51 -07001298 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001299 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001300 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001301 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1302 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001303}
1304
mflodmancc3d4422017-08-03 08:27:51 -07001305TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001306 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps),
1309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001310
1311 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001312 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001313 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001314 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1315 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001316 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1317 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001318 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001319 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001320
mflodmancc3d4422017-08-03 08:27:51 -07001321 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001322}
1323
Noah Richards51db4212019-06-12 06:59:12 -07001324TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1325 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001326 DataRate::BitsPerSec(kTargetBitrateBps),
1327 DataRate::BitsPerSec(kTargetBitrateBps),
1328 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001329
1330 rtc::Event frame_destroyed_event;
1331 video_source_.IncomingCapturedFrame(
1332 CreateFakeNativeFrame(1, &frame_destroyed_event));
1333 ExpectDroppedFrame();
1334 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1335 video_stream_encoder_->Stop();
1336}
1337
1338TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1339 // Use the cropping factory.
1340 video_encoder_config_.video_stream_factory =
1341 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1342 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1343 kMaxPayloadLength);
1344 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1345
1346 // Capture a frame at codec_width_/codec_height_.
1347 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001348 DataRate::BitsPerSec(kTargetBitrateBps),
1349 DataRate::BitsPerSec(kTargetBitrateBps),
1350 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001351 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1352 WaitForEncodedFrame(1);
1353 // The encoder will have been configured once.
1354 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1355 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1356 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1357
1358 // Now send in a fake frame that needs to be cropped as the width/height
1359 // aren't divisible by 4 (see CreateEncoderStreams above).
1360 rtc::Event frame_destroyed_event;
1361 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1362 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1363 ExpectDroppedFrame();
1364 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1365 video_stream_encoder_->Stop();
1366}
1367
Ying Wang9b881ab2020-02-07 14:29:32 +01001368TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
1369 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001370 DataRate::BitsPerSec(kTargetBitrateBps),
1371 DataRate::BitsPerSec(kTargetBitrateBps),
1372 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001373 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1374 WaitForEncodedFrame(1);
1375
1376 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001377 DataRate::BitsPerSec(kTargetBitrateBps),
1378 DataRate::BitsPerSec(kTargetBitrateBps),
1379 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001380 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1381 // frames. Adding two frames means that the first frame will be dropped and
1382 // the second frame will be sent to the encoder.
1383 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1384 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1385 WaitForEncodedFrame(3);
1386 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1387 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1388 WaitForEncodedFrame(5);
1389 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1390 video_stream_encoder_->Stop();
1391}
1392
mflodmancc3d4422017-08-03 08:27:51 -07001393TEST_F(VideoStreamEncoderTest,
1394 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001395 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001396 DataRate::BitsPerSec(kTargetBitrateBps),
1397 DataRate::BitsPerSec(kTargetBitrateBps),
1398 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001399 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001400
1401 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001402 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001403 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001404 // The encoder will have been configured once when the first frame is
1405 // received.
1406 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001407
1408 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001409 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001410 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001411 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001412 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001413
1414 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001415 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001416 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001417 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001418 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001419
mflodmancc3d4422017-08-03 08:27:51 -07001420 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001421}
1422
mflodmancc3d4422017-08-03 08:27:51 -07001423TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001424 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001425 DataRate::BitsPerSec(kTargetBitrateBps),
1426 DataRate::BitsPerSec(kTargetBitrateBps),
1427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001428
1429 // Capture a frame and wait for it to synchronize with the encoder thread.
1430 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001431 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001432 // The encoder will have been configured once.
1433 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001434 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1435 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1436
1437 codec_width_ *= 2;
1438 codec_height_ *= 2;
1439 // Capture a frame with a higher resolution and wait for it to synchronize
1440 // with the encoder thread.
1441 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001442 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001443 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1444 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001445 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001446
mflodmancc3d4422017-08-03 08:27:51 -07001447 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001448}
1449
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001450TEST_F(VideoStreamEncoderTest,
1451 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1452 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps),
1455 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001456
1457 // Capture a frame and wait for it to synchronize with the encoder thread.
1458 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1459 WaitForEncodedFrame(1);
1460
1461 VideoEncoderConfig video_encoder_config;
1462 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1463 // Changing the max payload data length recreates encoder.
1464 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1465 kMaxPayloadLength / 2);
1466
1467 // Capture a frame and wait for it to synchronize with the encoder thread.
1468 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1469 WaitForEncodedFrame(2);
1470 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1471
1472 video_stream_encoder_->Stop();
1473}
1474
Sergey Silkin5ee69672019-07-02 14:18:34 +02001475TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1476 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001477 DataRate::BitsPerSec(kTargetBitrateBps),
1478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001480
1481 VideoEncoderConfig video_encoder_config;
1482 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1483 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1484 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1485 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1486 kMaxPayloadLength);
1487
1488 // Capture a frame and wait for it to synchronize with the encoder thread.
1489 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1490 WaitForEncodedFrame(1);
1491 // The encoder will have been configured once when the first frame is
1492 // received.
1493 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1494 EXPECT_EQ(kTargetBitrateBps,
1495 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1496 EXPECT_EQ(kStartBitrateBps,
1497 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1498
Sergey Silkin6456e352019-07-08 17:56:40 +02001499 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1500 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001501 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1502 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1503 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1504 kMaxPayloadLength);
1505
1506 // Capture a frame and wait for it to synchronize with the encoder thread.
1507 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1508 WaitForEncodedFrame(2);
1509 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1510 // Bitrate limits have changed - rate allocator should be reconfigured,
1511 // encoder should not be reconfigured.
1512 EXPECT_EQ(kTargetBitrateBps * 2,
1513 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1514 EXPECT_EQ(kStartBitrateBps * 2,
1515 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1516 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1517
1518 video_stream_encoder_->Stop();
1519}
1520
Sergey Silkin6456e352019-07-08 17:56:40 +02001521TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001522 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001523 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001524 DataRate::BitsPerSec(kTargetBitrateBps),
1525 DataRate::BitsPerSec(kTargetBitrateBps),
1526 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001527
Sergey Silkincd02eba2020-01-20 14:48:40 +01001528 const uint32_t kMinEncBitrateKbps = 100;
1529 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001530 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001531 /*frame_size_pixels=*/codec_width_ * codec_height_,
1532 /*min_start_bitrate_bps=*/0,
1533 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1534 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001535 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1536
Sergey Silkincd02eba2020-01-20 14:48:40 +01001537 VideoEncoderConfig video_encoder_config;
1538 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1539 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1540 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1541 (kMinEncBitrateKbps + 1) * 1000;
1542 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1543 kMaxPayloadLength);
1544
1545 // When both encoder and app provide bitrate limits, the intersection of
1546 // provided sets should be used.
1547 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1548 WaitForEncodedFrame(1);
1549 EXPECT_EQ(kMaxEncBitrateKbps,
1550 bitrate_allocator_factory_.codec_config().maxBitrate);
1551 EXPECT_EQ(kMinEncBitrateKbps + 1,
1552 bitrate_allocator_factory_.codec_config().minBitrate);
1553
1554 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1555 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1556 (kMinEncBitrateKbps - 1) * 1000;
1557 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1558 kMaxPayloadLength);
1559 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001560 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001561 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001562 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001563 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001564 bitrate_allocator_factory_.codec_config().minBitrate);
1565
Sergey Silkincd02eba2020-01-20 14:48:40 +01001566 video_stream_encoder_->Stop();
1567}
1568
1569TEST_F(VideoStreamEncoderTest,
1570 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
1571 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001572 DataRate::BitsPerSec(kTargetBitrateBps),
1573 DataRate::BitsPerSec(kTargetBitrateBps),
1574 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001575
1576 const uint32_t kMinAppBitrateKbps = 100;
1577 const uint32_t kMaxAppBitrateKbps = 200;
1578 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1579 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1580 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1581 /*frame_size_pixels=*/codec_width_ * codec_height_,
1582 /*min_start_bitrate_bps=*/0,
1583 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1584 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1585 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1586
1587 VideoEncoderConfig video_encoder_config;
1588 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1589 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1590 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1591 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001592 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1593 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001594
Sergey Silkincd02eba2020-01-20 14:48:40 +01001595 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1596 WaitForEncodedFrame(1);
1597 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001598 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001599 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001600 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001601
1602 video_stream_encoder_->Stop();
1603}
1604
1605TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001606 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001607 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001608 DataRate::BitsPerSec(kTargetBitrateBps),
1609 DataRate::BitsPerSec(kTargetBitrateBps),
1610 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001611
1612 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001613 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001614 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001615 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001616 fake_encoder_.SetResolutionBitrateLimits(
1617 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1618
1619 VideoEncoderConfig video_encoder_config;
1620 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1621 video_encoder_config.max_bitrate_bps = 0;
1622 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1623 kMaxPayloadLength);
1624
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001625 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001626 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1627 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001628 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1629 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001630 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1631 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1632
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001633 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001634 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1635 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001636 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1637 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001638 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1639 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1640
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001641 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001642 // encoder for 360p should be used.
1643 video_source_.IncomingCapturedFrame(
1644 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1645 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001646 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1647 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001648 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1649 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1650
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001651 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001652 // ignored.
1653 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1654 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001655 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1656 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001657 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1658 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001659 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1660 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001661 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1662 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1663
1664 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1665 // for 270p should be used.
1666 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1667 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001668 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1669 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001670 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1671 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1672
1673 video_stream_encoder_->Stop();
1674}
1675
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001676TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1677 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001678 DataRate::BitsPerSec(kTargetBitrateBps),
1679 DataRate::BitsPerSec(kTargetBitrateBps),
1680 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001681
1682 VideoEncoderConfig video_encoder_config;
1683 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1684 video_encoder_config.max_bitrate_bps = 0;
1685 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1686 kMaxPayloadLength);
1687
1688 // Encode 720p frame to get the default encoder target bitrate.
1689 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1690 WaitForEncodedFrame(1);
1691 const uint32_t kDefaultTargetBitrateFor720pKbps =
1692 bitrate_allocator_factory_.codec_config()
1693 .simulcastStream[0]
1694 .targetBitrate;
1695
1696 // Set the max recommended encoder bitrate to something lower than the default
1697 // target bitrate.
1698 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1699 1280 * 720, 10 * 1000, 10 * 1000,
1700 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1701 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1702
1703 // Change resolution to trigger encoder reinitialization.
1704 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1705 WaitForEncodedFrame(2);
1706 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1707 WaitForEncodedFrame(3);
1708
1709 // Ensure the target bitrate is capped by the max bitrate.
1710 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1711 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1712 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1713 .simulcastStream[0]
1714 .targetBitrate *
1715 1000,
1716 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1717
1718 video_stream_encoder_->Stop();
1719}
1720
mflodmancc3d4422017-08-03 08:27:51 -07001721TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001722 EXPECT_TRUE(video_source_.has_sinks());
1723 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001724 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001725 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001726 EXPECT_FALSE(video_source_.has_sinks());
1727 EXPECT_TRUE(new_video_source.has_sinks());
1728
mflodmancc3d4422017-08-03 08:27:51 -07001729 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001730}
1731
mflodmancc3d4422017-08-03 08:27:51 -07001732TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001733 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001734 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001735 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001736 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001737}
1738
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001739TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1740 constexpr int kRequestedResolutionAlignment = 7;
1741 video_source_.set_adaptation_enabled(true);
1742 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
1743 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001744 DataRate::BitsPerSec(kTargetBitrateBps),
1745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001747
1748 // On the 1st frame, we should have initialized the encoder and
1749 // asked for its resolution requirements.
1750 video_source_.IncomingCapturedFrame(
1751 CreateFrame(1, codec_width_, codec_height_));
1752 WaitForEncodedFrame(1);
1753 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1754 kRequestedResolutionAlignment);
1755
1756 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1757 // (It's up the to the encoder to potentially drop the previous frame,
1758 // to avoid coding back-to-back keyframes.)
1759 video_source_.IncomingCapturedFrame(
1760 CreateFrame(2, codec_width_, codec_height_));
1761 WaitForEncodedFrame(2);
1762 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1763
1764 video_stream_encoder_->Stop();
1765}
1766
Jonathan Yubc771b72017-12-08 17:04:29 -08001767TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1768 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001769 const int kWidth = 1280;
1770 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001771
1772 // We rely on the automatic resolution adaptation, but we handle framerate
1773 // adaptation manually by mocking the stats proxy.
1774 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001775
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001776 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001777 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001778 DataRate::BitsPerSec(kTargetBitrateBps),
1779 DataRate::BitsPerSec(kTargetBitrateBps),
1780 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001781 video_stream_encoder_->SetSource(&video_source_,
1782 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001783 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1787
Jonathan Yubc771b72017-12-08 17:04:29 -08001788 // Adapt down as far as possible.
1789 rtc::VideoSinkWants last_wants;
1790 int64_t t = 1;
1791 int loop_count = 0;
1792 do {
1793 ++loop_count;
1794 last_wants = video_source_.sink_wants();
1795
1796 // Simulate the framerate we've been asked to adapt to.
1797 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1798 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1799 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1800 mock_stats.input_frame_rate = fps;
1801 stats_proxy_->SetMockStats(mock_stats);
1802
1803 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1804 sink_.WaitForEncodedFrame(t);
1805 t += frame_interval_ms;
1806
mflodmancc3d4422017-08-03 08:27:51 -07001807 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001808 VerifyBalancedModeFpsRange(
1809 video_source_.sink_wants(),
1810 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1811 } while (video_source_.sink_wants().max_pixel_count <
1812 last_wants.max_pixel_count ||
1813 video_source_.sink_wants().max_framerate_fps <
1814 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001815
Jonathan Yubc771b72017-12-08 17:04:29 -08001816 // Verify that we've adapted all the way down.
1817 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001818 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001819 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1820 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001821 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001822 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1823 *video_source_.last_sent_height());
1824 EXPECT_EQ(kMinBalancedFramerateFps,
1825 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001826
Jonathan Yubc771b72017-12-08 17:04:29 -08001827 // Adapt back up the same number of times we adapted down.
1828 for (int i = 0; i < loop_count - 1; ++i) {
1829 last_wants = video_source_.sink_wants();
1830
1831 // Simulate the framerate we've been asked to adapt to.
1832 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1833 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1834 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1835 mock_stats.input_frame_rate = fps;
1836 stats_proxy_->SetMockStats(mock_stats);
1837
1838 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1839 sink_.WaitForEncodedFrame(t);
1840 t += frame_interval_ms;
1841
Henrik Boström91aa7322020-04-28 12:24:33 +02001842 video_stream_encoder_->TriggerCpuUnderuse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001843 VerifyBalancedModeFpsRange(
1844 video_source_.sink_wants(),
1845 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1846 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1847 last_wants.max_pixel_count ||
1848 video_source_.sink_wants().max_framerate_fps >
1849 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001850 }
1851
Åsa Persson8c1bf952018-09-13 10:42:19 +02001852 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001853 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1856 EXPECT_EQ((loop_count - 1) * 2,
1857 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001858
mflodmancc3d4422017-08-03 08:27:51 -07001859 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001860}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001861
mflodmancc3d4422017-08-03 08:27:51 -07001862TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001863 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps),
1866 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001867 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001868
sprangc5d62e22017-04-02 23:53:04 -07001869 const int kFrameWidth = 1280;
1870 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001871
Åsa Persson8c1bf952018-09-13 10:42:19 +02001872 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001873
kthelgason5e13d412016-12-01 03:59:51 -08001874 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001875 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001876 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001877 frame_timestamp += kFrameIntervalMs;
1878
perkj803d97f2016-11-01 11:45:46 -07001879 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001880 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001881 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001882 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001883 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001884 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001885
asapersson0944a802017-04-07 00:57:58 -07001886 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001887 // wanted resolution.
1888 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1889 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1890 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001891 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001892
1893 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001894 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001895 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001896 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001897 // Give the encoder queue time to process the change in degradation preference
1898 // by waiting for an encoded frame.
1899 new_video_source.IncomingCapturedFrame(
1900 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1901 sink_.WaitForEncodedFrame(frame_timestamp);
1902 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001903 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001904 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001905
sprangc5d62e22017-04-02 23:53:04 -07001906 // Force an input frame rate to be available, or the adaptation call won't
1907 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001908 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001909 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001910 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001911 stats_proxy_->SetMockStats(stats);
1912
mflodmancc3d4422017-08-03 08:27:51 -07001913 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001914 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001915 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001916 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001917 frame_timestamp += kFrameIntervalMs;
1918
1919 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001920 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001921 EXPECT_EQ(std::numeric_limits<int>::max(),
1922 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001923 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001924
asapersson02465b82017-04-10 01:12:52 -07001925 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001926 video_stream_encoder_->SetSource(&new_video_source,
1927 webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01001928 // Give the encoder queue time to process the change in degradation preference
1929 // by waiting for an encoded frame.
1930 new_video_source.IncomingCapturedFrame(
1931 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1932 sink_.WaitForEncodedFrame(frame_timestamp);
1933 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001934 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001935
mflodmancc3d4422017-08-03 08:27:51 -07001936 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001937 new_video_source.IncomingCapturedFrame(
1938 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001939 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001940 frame_timestamp += kFrameIntervalMs;
1941
1942 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001943 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001944
1945 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001946 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001947 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01001948 // Give the encoder queue time to process the change in degradation preference
1949 // by waiting for an encoded frame.
1950 new_video_source.IncomingCapturedFrame(
1951 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1952 sink_.WaitForEncodedFrame(frame_timestamp);
1953 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001954 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1955 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001956 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001957 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001958
1959 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001960 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001961 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001962 // Give the encoder queue time to process the change in degradation preference
1963 // by waiting for an encoded frame.
1964 new_video_source.IncomingCapturedFrame(
1965 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1966 sink_.WaitForEncodedFrame(frame_timestamp);
1967 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001968 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1969 EXPECT_EQ(std::numeric_limits<int>::max(),
1970 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001971 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001972
mflodmancc3d4422017-08-03 08:27:51 -07001973 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001974}
1975
mflodmancc3d4422017-08-03 08:27:51 -07001976TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001977 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001978 DataRate::BitsPerSec(kTargetBitrateBps),
1979 DataRate::BitsPerSec(kTargetBitrateBps),
1980 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001981
asaperssonfab67072017-04-04 05:51:49 -07001982 const int kWidth = 1280;
1983 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001984 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001985 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001986 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1987 EXPECT_FALSE(stats.bw_limited_resolution);
1988 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1989
1990 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001991 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001992 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001993 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001994
1995 stats = stats_proxy_->GetStats();
1996 EXPECT_TRUE(stats.bw_limited_resolution);
1997 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1998
1999 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002000 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002001 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002002 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002003
2004 stats = stats_proxy_->GetStats();
2005 EXPECT_FALSE(stats.bw_limited_resolution);
2006 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2007 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2008
mflodmancc3d4422017-08-03 08:27:51 -07002009 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002010}
2011
mflodmancc3d4422017-08-03 08:27:51 -07002012TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002013 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002014 DataRate::BitsPerSec(kTargetBitrateBps),
2015 DataRate::BitsPerSec(kTargetBitrateBps),
2016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002017
2018 const int kWidth = 1280;
2019 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002020 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002021 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002022 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2023 EXPECT_FALSE(stats.cpu_limited_resolution);
2024 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2025
2026 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002027 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002028 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002029 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002030
2031 stats = stats_proxy_->GetStats();
2032 EXPECT_TRUE(stats.cpu_limited_resolution);
2033 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2034
2035 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002036 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002037 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002038 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002039
2040 stats = stats_proxy_->GetStats();
2041 EXPECT_FALSE(stats.cpu_limited_resolution);
2042 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002043 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002044
mflodmancc3d4422017-08-03 08:27:51 -07002045 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002046}
2047
mflodmancc3d4422017-08-03 08:27:51 -07002048TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002049 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002050 DataRate::BitsPerSec(kTargetBitrateBps),
2051 DataRate::BitsPerSec(kTargetBitrateBps),
2052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002053
asaperssonfab67072017-04-04 05:51:49 -07002054 const int kWidth = 1280;
2055 const int kHeight = 720;
2056 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002057 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002058 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002059 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002060 EXPECT_FALSE(stats.cpu_limited_resolution);
2061 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2062
asaperssonfab67072017-04-04 05:51:49 -07002063 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002064 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002065 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002066 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002067 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002068 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002069 EXPECT_TRUE(stats.cpu_limited_resolution);
2070 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2071
2072 // Set new source with adaptation still enabled.
2073 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002074 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002075 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002076
asaperssonfab67072017-04-04 05:51:49 -07002077 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002078 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002079 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002080 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002081 EXPECT_TRUE(stats.cpu_limited_resolution);
2082 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2083
2084 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002085 video_stream_encoder_->SetSource(&new_video_source,
2086 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002087
asaperssonfab67072017-04-04 05:51:49 -07002088 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002089 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002090 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002091 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002092 EXPECT_FALSE(stats.cpu_limited_resolution);
2093 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2094
2095 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002096 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002097 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002098
asaperssonfab67072017-04-04 05:51:49 -07002099 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002100 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002101 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002102 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002103 EXPECT_TRUE(stats.cpu_limited_resolution);
2104 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2105
asaperssonfab67072017-04-04 05:51:49 -07002106 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002107 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002108 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002109 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002110 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002111 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002112 EXPECT_FALSE(stats.cpu_limited_resolution);
2113 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002114 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002115
mflodmancc3d4422017-08-03 08:27:51 -07002116 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002117}
2118
mflodmancc3d4422017-08-03 08:27:51 -07002119TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002120 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002121 DataRate::BitsPerSec(kTargetBitrateBps),
2122 DataRate::BitsPerSec(kTargetBitrateBps),
2123 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002124
asaperssonfab67072017-04-04 05:51:49 -07002125 const int kWidth = 1280;
2126 const int kHeight = 720;
2127 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002128 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002129 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002130 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002131 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002132 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002133
2134 // Set new source with adaptation still enabled.
2135 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002136 video_stream_encoder_->SetSource(&new_video_source,
2137 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002138
asaperssonfab67072017-04-04 05:51:49 -07002139 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002140 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002141 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002142 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002143 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002144 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002145
asaperssonfab67072017-04-04 05:51:49 -07002146 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002147 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002148 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002149 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002150 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002151 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002152 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002153 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002154
asaperssonfab67072017-04-04 05:51:49 -07002155 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002156 video_stream_encoder_->SetSource(&new_video_source,
2157 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002158
asaperssonfab67072017-04-04 05:51:49 -07002159 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002160 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002161 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002162 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002163 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002164 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002165
asapersson02465b82017-04-10 01:12:52 -07002166 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002168 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002169
asaperssonfab67072017-04-04 05:51:49 -07002170 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002171 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002172 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002173 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002174 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002175 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2176 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002177
mflodmancc3d4422017-08-03 08:27:51 -07002178 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002179}
2180
mflodmancc3d4422017-08-03 08:27:51 -07002181TEST_F(VideoStreamEncoderTest,
2182 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002183 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002184 DataRate::BitsPerSec(kTargetBitrateBps),
2185 DataRate::BitsPerSec(kTargetBitrateBps),
2186 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002187
2188 const int kWidth = 1280;
2189 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002190 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002191 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002192 video_source_.IncomingCapturedFrame(
2193 CreateFrame(timestamp_ms, kWidth, kHeight));
2194 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002195 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2196 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2197 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2198
2199 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002200 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002201 timestamp_ms += kFrameIntervalMs;
2202 video_source_.IncomingCapturedFrame(
2203 CreateFrame(timestamp_ms, kWidth, kHeight));
2204 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002205 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2206 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2207 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2208
2209 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002210 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002211 timestamp_ms += kFrameIntervalMs;
2212 video_source_.IncomingCapturedFrame(
2213 CreateFrame(timestamp_ms, kWidth, kHeight));
2214 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002215 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2216 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2217 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2218
Niels Möller4db138e2018-04-19 09:04:13 +02002219 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002220 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002221
2222 VideoEncoderConfig video_encoder_config;
2223 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2224 // Make format different, to force recreation of encoder.
2225 video_encoder_config.video_format.parameters["foo"] = "foo";
2226 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002227 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002228 timestamp_ms += kFrameIntervalMs;
2229 video_source_.IncomingCapturedFrame(
2230 CreateFrame(timestamp_ms, kWidth, kHeight));
2231 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002232 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2234 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2235
mflodmancc3d4422017-08-03 08:27:51 -07002236 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002237}
2238
mflodmancc3d4422017-08-03 08:27:51 -07002239TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002240 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
2241 video_stream_encoder_->OnBitrateUpdated(
2242 DataRate::BitsPerSec(kTargetBitrateBps),
2243 DataRate::BitsPerSec(kTargetBitrateBps),
2244 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2245
2246 const int kWidth = 1280;
2247 const int kHeight = 720;
2248 int sequence = 1;
2249
2250 // Enable BALANCED preference, no initial limitation.
2251 test::FrameForwarder source;
2252 video_stream_encoder_->SetSource(&source,
2253 webrtc::DegradationPreference::BALANCED);
2254 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2255 WaitForEncodedFrame(sequence++);
2256 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2257 EXPECT_FALSE(stats.cpu_limited_resolution);
2258 EXPECT_FALSE(stats.cpu_limited_framerate);
2259 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2260
2261 // Trigger CPU overuse, should now adapt down.
2262 video_stream_encoder_->TriggerCpuOveruse();
2263 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2264 WaitForEncodedFrame(sequence++);
2265 stats = stats_proxy_->GetStats();
2266 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2267
2268 // Set new degradation preference should clear restrictions since we changed
2269 // from BALANCED.
2270 video_stream_encoder_->SetSource(
2271 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2272 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2273 WaitForEncodedFrame(sequence++);
2274 stats = stats_proxy_->GetStats();
2275 EXPECT_FALSE(stats.cpu_limited_resolution);
2276 EXPECT_FALSE(stats.cpu_limited_framerate);
2277 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2278
2279 // Force an input frame rate to be available, or the adaptation call won't
2280 // know what framerate to adapt from.
2281 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2282 mock_stats.input_frame_rate = 30;
2283 stats_proxy_->SetMockStats(mock_stats);
2284 video_stream_encoder_->TriggerCpuOveruse();
2285 stats_proxy_->ResetMockStats();
2286 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2287 WaitForEncodedFrame(sequence++);
2288
2289 // We have now adapted once.
2290 stats = stats_proxy_->GetStats();
2291 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2292
2293 // Back to BALANCED, should clear the restrictions again.
2294 video_stream_encoder_->SetSource(&source,
2295 webrtc::DegradationPreference::BALANCED);
2296 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2297 WaitForEncodedFrame(sequence++);
2298 stats = stats_proxy_->GetStats();
2299 EXPECT_FALSE(stats.cpu_limited_resolution);
2300 EXPECT_FALSE(stats.cpu_limited_framerate);
2301 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2302
2303 video_stream_encoder_->Stop();
2304}
2305
2306TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002307 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002308 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002309 DataRate::BitsPerSec(kTargetBitrateBps),
2310 DataRate::BitsPerSec(kTargetBitrateBps),
2311 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002312
asapersson0944a802017-04-07 00:57:58 -07002313 const int kWidth = 1280;
2314 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002315 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002316
asaperssonfab67072017-04-04 05:51:49 -07002317 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002318 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002319 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002320 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002321 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002322 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2323
asapersson02465b82017-04-10 01:12:52 -07002324 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002325 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002326 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002327 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002328 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002329 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002330 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002331 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2332
2333 // Set new source with adaptation still enabled.
2334 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002335 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002336 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002337
2338 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002339 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002340 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002341 stats = stats_proxy_->GetStats();
2342 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002343 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002344 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2345
sprangc5d62e22017-04-02 23:53:04 -07002346 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002347 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002348 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002349 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002350 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002351 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002352 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002353 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002354 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002355 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002356 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2357
sprangc5d62e22017-04-02 23:53:04 -07002358 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002359 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002360 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2361 mock_stats.input_frame_rate = 30;
2362 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002363 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002364 stats_proxy_->ResetMockStats();
2365
2366 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002367 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002368 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002369
2370 // Framerate now adapted.
2371 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002372 EXPECT_FALSE(stats.cpu_limited_resolution);
2373 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002374 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2375
2376 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002377 video_stream_encoder_->SetSource(&new_video_source,
2378 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002379 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002380 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002381 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002382
2383 stats = stats_proxy_->GetStats();
2384 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002385 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002386 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2387
2388 // Try to trigger overuse. Should not succeed.
2389 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002390 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002391 stats_proxy_->ResetMockStats();
2392
2393 stats = stats_proxy_->GetStats();
2394 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002395 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002396 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2397
2398 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002399 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002400 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002401 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002402 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002403 stats = stats_proxy_->GetStats();
2404 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002405 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002406 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002407
2408 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002409 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002410 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002411 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002412 stats = stats_proxy_->GetStats();
2413 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002414 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002415 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2416
2417 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002418 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002419 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002420 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002421 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002422 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002423 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002424 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002425 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002426 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002427 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2428
2429 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002430 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002431 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002432 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002433 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002434 stats = stats_proxy_->GetStats();
2435 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002436 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002437 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002438 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002439
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002441}
2442
mflodmancc3d4422017-08-03 08:27:51 -07002443TEST_F(VideoStreamEncoderTest,
2444 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002445 const int kWidth = 1280;
2446 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002447 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002448 DataRate::BitsPerSec(kTargetBitrateBps),
2449 DataRate::BitsPerSec(kTargetBitrateBps),
2450 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002451
asaperssonfab67072017-04-04 05:51:49 -07002452 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002453 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002454
asaperssonfab67072017-04-04 05:51:49 -07002455 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002456 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002457
asaperssonfab67072017-04-04 05:51:49 -07002458 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002459 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002460
asaperssonfab67072017-04-04 05:51:49 -07002461 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002462 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002463
kthelgason876222f2016-11-29 01:44:11 -08002464 // Expect a scale down.
2465 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002466 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002467
asapersson02465b82017-04-10 01:12:52 -07002468 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002469 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002470 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002471 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002472
asaperssonfab67072017-04-04 05:51:49 -07002473 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002474 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002475 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002476 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002477
asaperssonfab67072017-04-04 05:51:49 -07002478 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002479 EXPECT_EQ(std::numeric_limits<int>::max(),
2480 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002481
asaperssonfab67072017-04-04 05:51:49 -07002482 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002483 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002484 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002485 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002486
asapersson02465b82017-04-10 01:12:52 -07002487 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002488 EXPECT_EQ(std::numeric_limits<int>::max(),
2489 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002490
mflodmancc3d4422017-08-03 08:27:51 -07002491 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002492}
2493
mflodmancc3d4422017-08-03 08:27:51 -07002494TEST_F(VideoStreamEncoderTest,
2495 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002496 const int kWidth = 1280;
2497 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002498 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002499 DataRate::BitsPerSec(kTargetBitrateBps),
2500 DataRate::BitsPerSec(kTargetBitrateBps),
2501 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002502
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002503 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002504 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002505 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002506 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002507
2508 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002509 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002510 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002511 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2512 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2513
2514 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002515 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002516 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002517 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2518 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2519 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2520
2521 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002522 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002523 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2524 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2525 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2526
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002528}
2529
mflodmancc3d4422017-08-03 08:27:51 -07002530TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002531 const int kWidth = 1280;
2532 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002533 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002534 DataRate::BitsPerSec(kTargetBitrateBps),
2535 DataRate::BitsPerSec(kTargetBitrateBps),
2536 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002537
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002538 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002539 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002540 video_stream_encoder_->SetSource(&source,
2541 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002542 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2543 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002544 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002545
2546 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002547 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002548 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2550 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2551 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2552
2553 // Trigger adapt down for same input resolution, expect no change.
2554 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2555 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002556 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002557 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2558 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2559 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2560
2561 // Trigger adapt down for larger input resolution, expect no change.
2562 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2563 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002564 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002565 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2567 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2568
mflodmancc3d4422017-08-03 08:27:51 -07002569 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002570}
2571
mflodmancc3d4422017-08-03 08:27:51 -07002572TEST_F(VideoStreamEncoderTest,
2573 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002574 const int kWidth = 1280;
2575 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002576 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002577 DataRate::BitsPerSec(kTargetBitrateBps),
2578 DataRate::BitsPerSec(kTargetBitrateBps),
2579 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002580
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002581 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002582 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002583 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002584 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002585
2586 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002587 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002588 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002589 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2590 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2591
2592 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002593 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002594 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002595 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2596 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2597
mflodmancc3d4422017-08-03 08:27:51 -07002598 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002599}
2600
mflodmancc3d4422017-08-03 08:27:51 -07002601TEST_F(VideoStreamEncoderTest,
2602 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002603 const int kWidth = 1280;
2604 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002605 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002606 DataRate::BitsPerSec(kTargetBitrateBps),
2607 DataRate::BitsPerSec(kTargetBitrateBps),
2608 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002609
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002610 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002611 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002612 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002613 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002614
2615 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002616 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002617 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002618 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002619 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2620
2621 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002622 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002623 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002624 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002625 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2626
mflodmancc3d4422017-08-03 08:27:51 -07002627 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002628}
2629
mflodmancc3d4422017-08-03 08:27:51 -07002630TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002631 const int kWidth = 1280;
2632 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002633 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002634 DataRate::BitsPerSec(kTargetBitrateBps),
2635 DataRate::BitsPerSec(kTargetBitrateBps),
2636 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002637
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002638 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002639 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002640 video_stream_encoder_->SetSource(&source,
2641 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002642
2643 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2644 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002645 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002646 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2647 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2648 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2649
2650 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002651 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002652 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2654 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2655 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2656
mflodmancc3d4422017-08-03 08:27:51 -07002657 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002658}
2659
mflodmancc3d4422017-08-03 08:27:51 -07002660TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002661 const int kWidth = 1280;
2662 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002663 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002664 DataRate::BitsPerSec(kTargetBitrateBps),
2665 DataRate::BitsPerSec(kTargetBitrateBps),
2666 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002667
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002668 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002669 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002670 video_stream_encoder_->SetSource(&source,
2671 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002672
2673 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2674 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002675 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2678 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2679
2680 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002681 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002682 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002683 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2685 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2686
mflodmancc3d4422017-08-03 08:27:51 -07002687 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002688}
2689
mflodmancc3d4422017-08-03 08:27:51 -07002690TEST_F(VideoStreamEncoderTest,
2691 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002692 const int kWidth = 1280;
2693 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002694 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002695 DataRate::BitsPerSec(kTargetBitrateBps),
2696 DataRate::BitsPerSec(kTargetBitrateBps),
2697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002698
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002699 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002700 AdaptingFrameForwarder source;
2701 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002702 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002703 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002704
2705 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002706 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002707 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2709 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2710
2711 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002712 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002713 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002714 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002715 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002716 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2717 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2718
2719 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002720 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002721 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2723 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2725
mflodmancc3d4422017-08-03 08:27:51 -07002726 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002727}
2728
mflodmancc3d4422017-08-03 08:27:51 -07002729TEST_F(VideoStreamEncoderTest,
2730 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002731 const int kWidth = 1280;
2732 const int kHeight = 720;
2733 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002734 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002735 DataRate::BitsPerSec(kTargetBitrateBps),
2736 DataRate::BitsPerSec(kTargetBitrateBps),
2737 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002738
2739 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2740 stats.input_frame_rate = kInputFps;
2741 stats_proxy_->SetMockStats(stats);
2742
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002743 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002744 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2745 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002746 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002747
2748 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002749 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002750 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2751 sink_.WaitForEncodedFrame(2);
2752 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2753
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002754 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002755 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002756 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002757 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002758 // Give the encoder queue time to process the change in degradation preference
2759 // by waiting for an encoded frame.
2760 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2761 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002762 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002763
2764 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002765 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002766 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2767 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002768 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2769
2770 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002771 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002772 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002773
mflodmancc3d4422017-08-03 08:27:51 -07002774 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002775}
2776
mflodmancc3d4422017-08-03 08:27:51 -07002777TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002778 const int kWidth = 1280;
2779 const int kHeight = 720;
2780 const size_t kNumFrames = 10;
2781
Erik Språng4c6ca302019-04-08 15:14:01 +02002782 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002783 DataRate::BitsPerSec(kTargetBitrateBps),
2784 DataRate::BitsPerSec(kTargetBitrateBps),
2785 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002786
asaperssond0de2952017-04-21 01:47:31 -07002787 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002788 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002789 video_source_.set_adaptation_enabled(true);
2790
2791 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2793
2794 int downscales = 0;
2795 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002796 video_source_.IncomingCapturedFrame(
2797 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2798 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002799
asaperssonfab67072017-04-04 05:51:49 -07002800 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002801 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002802 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002803 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002804
2805 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2806 ++downscales;
2807
2808 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2809 EXPECT_EQ(downscales,
2810 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2811 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002812 }
mflodmancc3d4422017-08-03 08:27:51 -07002813 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002814}
2815
mflodmancc3d4422017-08-03 08:27:51 -07002816TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002817 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2818 const int kWidth = 1280;
2819 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002820 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002821 DataRate::BitsPerSec(kTargetBitrateBps),
2822 DataRate::BitsPerSec(kTargetBitrateBps),
2823 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002824
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002825 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002826 AdaptingFrameForwarder source;
2827 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002828 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002829 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002830
Åsa Persson8c1bf952018-09-13 10:42:19 +02002831 int64_t timestamp_ms = kFrameIntervalMs;
2832 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002833 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002834 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002835 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2836 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2837
2838 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002839 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002840 timestamp_ms += kFrameIntervalMs;
2841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2842 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002843 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002844 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2845 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2846
2847 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02002848 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002849 timestamp_ms += kFrameIntervalMs;
2850 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002851 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002852 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2854 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2855
2856 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002858 timestamp_ms += kFrameIntervalMs;
2859 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2860 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002861 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002862 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2863 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2864
2865 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02002866 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002867 timestamp_ms += kFrameIntervalMs;
2868 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002869 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002870 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2872 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2873
mflodmancc3d4422017-08-03 08:27:51 -07002874 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002875}
2876
mflodmancc3d4422017-08-03 08:27:51 -07002877TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002878 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2879 const int kWidth = 1280;
2880 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002881 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002882 DataRate::BitsPerSec(kTargetBitrateBps),
2883 DataRate::BitsPerSec(kTargetBitrateBps),
2884 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002885
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002886 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002887 AdaptingFrameForwarder source;
2888 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002889 video_stream_encoder_->SetSource(&source,
2890 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002891
Åsa Persson8c1bf952018-09-13 10:42:19 +02002892 int64_t timestamp_ms = kFrameIntervalMs;
2893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002894 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002895 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2897 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2898
2899 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002900 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002901 timestamp_ms += kFrameIntervalMs;
2902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2903 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002904 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2907
2908 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002910 timestamp_ms += kFrameIntervalMs;
2911 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002912 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002913 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002914 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2915 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2916
2917 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002918 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002919 timestamp_ms += kFrameIntervalMs;
2920 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2921 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002922 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2923 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2924 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2925
2926 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002927 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002928 timestamp_ms += kFrameIntervalMs;
2929 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002930 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002931 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002932 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2933 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2934
mflodmancc3d4422017-08-03 08:27:51 -07002935 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002936}
2937
Sergey Silkin41c650b2019-10-14 13:12:19 +02002938TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2939 fake_encoder_.SetResolutionBitrateLimits(
2940 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2941
2942 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002943 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2944 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2945 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2946 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002947
2948 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2949 AdaptingFrameForwarder source;
2950 source.set_adaptation_enabled(true);
2951 video_stream_encoder_->SetSource(
2952 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2953
2954 // Insert 720p frame.
2955 int64_t timestamp_ms = kFrameIntervalMs;
2956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2957 WaitForEncodedFrame(1280, 720);
2958
2959 // Reduce bitrate and trigger adapt down.
2960 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002961 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2962 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2963 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2964 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002965 video_stream_encoder_->TriggerQualityLow();
2966
2967 // Insert 720p frame. It should be downscaled and encoded.
2968 timestamp_ms += kFrameIntervalMs;
2969 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2970 WaitForEncodedFrame(960, 540);
2971
2972 // Trigger adapt up. Higher resolution should not be requested duo to lack
2973 // of bitrate.
2974 video_stream_encoder_->TriggerQualityHigh();
2975 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2976
2977 // Increase bitrate.
2978 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002979 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2980 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2981 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2982 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002983
2984 // Trigger adapt up. Higher resolution should be requested.
2985 video_stream_encoder_->TriggerQualityHigh();
2986 VerifyFpsMaxResolutionMax(source.sink_wants());
2987
2988 video_stream_encoder_->Stop();
2989}
2990
2991TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2992 fake_encoder_.SetResolutionBitrateLimits(
2993 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2994
2995 // Set bitrate equal to min bitrate of 540p.
2996 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002997 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2998 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2999 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3000 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003001
3002 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3003 AdaptingFrameForwarder source;
3004 source.set_adaptation_enabled(true);
3005 video_stream_encoder_->SetSource(
3006 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3007
3008 // Insert 720p frame. It should be dropped and lower resolution should be
3009 // requested.
3010 int64_t timestamp_ms = kFrameIntervalMs;
3011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3012 ExpectDroppedFrame();
3013 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3014
3015 // Insert 720p frame. It should be downscaled and encoded.
3016 timestamp_ms += kFrameIntervalMs;
3017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3018 WaitForEncodedFrame(960, 540);
3019
3020 video_stream_encoder_->Stop();
3021}
3022
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003023class BalancedDegradationTest : public VideoStreamEncoderTest {
3024 protected:
3025 void SetupTest() {
3026 // Reset encoder for field trials to take effect.
3027 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003028 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003029
3030 // Enable BALANCED preference.
3031 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003032 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3033 }
3034
3035 void OnBitrateUpdated(int bitrate_bps) {
Ying Wang9b881ab2020-02-07 14:29:32 +01003036 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003037 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3038 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003039 }
3040
Åsa Persson45b176f2019-09-30 11:19:05 +02003041 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003042 timestamp_ms_ += kFrameIntervalMs;
3043 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003044 }
3045
3046 void InsertFrameAndWaitForEncoded() {
3047 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003048 sink_.WaitForEncodedFrame(timestamp_ms_);
3049 }
3050
3051 const int kWidth = 640; // pixels:640x360=230400
3052 const int kHeight = 360;
3053 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3054 int64_t timestamp_ms_ = 0;
3055 AdaptingFrameForwarder source_;
3056};
3057
3058TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3059 test::ScopedFieldTrials field_trials(
3060 "WebRTC-Video-BalancedDegradationSettings/"
3061 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3062 SetupTest();
3063
3064 // Force input frame rate.
3065 const int kInputFps = 24;
3066 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3067 stats.input_frame_rate = kInputFps;
3068 stats_proxy_->SetMockStats(stats);
3069
Åsa Persson45b176f2019-09-30 11:19:05 +02003070 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003071 VerifyFpsMaxResolutionMax(source_.sink_wants());
3072
3073 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003074 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3075 // QP samples.
3076 EXPECT_FALSE(
3077 video_stream_encoder_
3078 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003079 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3080
3081 video_stream_encoder_->Stop();
3082}
3083
3084TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3085 test::ScopedFieldTrials field_trials(
3086 "WebRTC-Video-BalancedDegradationSettings/"
3087 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3088 SetupTest();
3089
3090 // Force input frame rate.
3091 const int kInputFps = 25;
3092 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3093 stats.input_frame_rate = kInputFps;
3094 stats_proxy_->SetMockStats(stats);
3095
Åsa Persson45b176f2019-09-30 11:19:05 +02003096 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003097 VerifyFpsMaxResolutionMax(source_.sink_wants());
3098
3099 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003100 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3101 // samples.
3102 EXPECT_TRUE(
3103 video_stream_encoder_
3104 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003105 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3106
3107 video_stream_encoder_->Stop();
3108}
3109
3110TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3111 test::ScopedFieldTrials field_trials(
3112 "WebRTC-Video-BalancedDegradationSettings/"
3113 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3114 SetupTest();
3115
3116 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3117
Åsa Persson45b176f2019-09-30 11:19:05 +02003118 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003119 VerifyFpsMaxResolutionMax(source_.sink_wants());
3120
3121 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3122 video_stream_encoder_->TriggerQualityLow();
3123 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3124
3125 video_stream_encoder_->Stop();
3126}
3127
Åsa Perssonccfb3402019-09-25 15:13:04 +02003128TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003129 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003130 "WebRTC-Video-BalancedDegradationSettings/"
3131 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003132 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003133
Åsa Persson1b247f12019-08-14 17:26:39 +02003134 const int kMinBitrateBps = 425000;
3135 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003136 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003137
Åsa Persson45b176f2019-09-30 11:19:05 +02003138 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003139 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003140 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3141
3142 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3143 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003144 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003145 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003146 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3147
3148 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3149 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003150 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003151 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003152 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3153
Åsa Persson30ab0152019-08-27 12:22:33 +02003154 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3155 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003156 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003157 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3158 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003159 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3160
3161 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003162 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003163 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003164 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003165
Åsa Persson30ab0152019-08-27 12:22:33 +02003166 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003167 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003168 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003169 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003170 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003171 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3172
3173 video_stream_encoder_->Stop();
3174}
3175
Åsa Perssonccfb3402019-09-25 15:13:04 +02003176TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003177 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3178 test::ScopedFieldTrials field_trials(
3179 "WebRTC-Video-BalancedDegradationSettings/"
3180 "pixels:57600|129600|230400,fps:7|24|24/");
3181 SetupTest();
3182 OnBitrateUpdated(kLowTargetBitrateBps);
3183
3184 VerifyNoLimitation(source_.sink_wants());
3185
3186 // Insert frame, expect scaled down:
3187 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3188 InsertFrame();
3189 EXPECT_FALSE(WaitForFrame(1000));
3190 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3191 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3192
3193 // Insert frame, expect scaled down:
3194 // resolution (320x180@24fps).
3195 InsertFrame();
3196 EXPECT_FALSE(WaitForFrame(1000));
3197 EXPECT_LT(source_.sink_wants().max_pixel_count,
3198 source_.last_wants().max_pixel_count);
3199 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3200
3201 // Frame should not be dropped (min pixels per frame reached).
3202 InsertFrameAndWaitForEncoded();
3203
3204 video_stream_encoder_->Stop();
3205}
3206
3207TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003208 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003209 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003210 "WebRTC-Video-BalancedDegradationSettings/"
3211 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003212 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003213
Åsa Persson30ab0152019-08-27 12:22:33 +02003214 const int kResolutionMinBitrateBps = 435000;
3215 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003216 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003217
Åsa Persson45b176f2019-09-30 11:19:05 +02003218 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003219 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003220 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3221
3222 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3223 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003224 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003225 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003226 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3227
3228 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3229 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003230 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003231 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003232 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3233
3234 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3235 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003236 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003237 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003238 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3239
Åsa Persson30ab0152019-08-27 12:22:33 +02003240 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3241 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003242 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003243 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003244 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3245
3246 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3247 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003248 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003249 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3250
3251 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003252 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003253 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003254 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003255 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003256 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3257
3258 video_stream_encoder_->Stop();
3259}
3260
Åsa Perssonccfb3402019-09-25 15:13:04 +02003261TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003262 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003263 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003264 "WebRTC-Video-BalancedDegradationSettings/"
3265 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003266 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003267
Åsa Persson30ab0152019-08-27 12:22:33 +02003268 const int kMinBitrateBps = 425000;
3269 const int kTooLowMinBitrateBps = 424000;
3270 const int kResolutionMinBitrateBps = 435000;
3271 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003272 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003273
Åsa Persson45b176f2019-09-30 11:19:05 +02003274 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003275 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003276 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3277
3278 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3279 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003280 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003281 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003282 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3283
3284 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3285 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003286 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003287 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003288 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3289
3290 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3291 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003292 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003293 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003294 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3295
3296 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3297 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003298 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003299 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3300
3301 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003302 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003303 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003304 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003305 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003306 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3307
3308 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003309 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003310 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003311 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003312 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3313
3314 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003315 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003316 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003317 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003318 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003319 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3320
Åsa Persson1b247f12019-08-14 17:26:39 +02003321 video_stream_encoder_->Stop();
3322}
3323
mflodmancc3d4422017-08-03 08:27:51 -07003324TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003325 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3326 const int kWidth = 1280;
3327 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02003328 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003329 DataRate::BitsPerSec(kTargetBitrateBps),
3330 DataRate::BitsPerSec(kTargetBitrateBps),
3331 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003332
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003333 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003334 AdaptingFrameForwarder source;
3335 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003336 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003337 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003338
Åsa Persson8c1bf952018-09-13 10:42:19 +02003339 int64_t timestamp_ms = kFrameIntervalMs;
3340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003341 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003342 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003343 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3345 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3347
3348 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003349 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003350 timestamp_ms += kFrameIntervalMs;
3351 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3352 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003353 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003354 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3355 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3356 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3357 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3358
3359 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003360 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003361 timestamp_ms += kFrameIntervalMs;
3362 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3363 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003364 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003365 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3366 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3367 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3368 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3369
Jonathan Yubc771b72017-12-08 17:04:29 -08003370 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003371 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003372 timestamp_ms += kFrameIntervalMs;
3373 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3374 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003375 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003376 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3377 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003378 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003379 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3380
Jonathan Yubc771b72017-12-08 17:04:29 -08003381 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003383 timestamp_ms += kFrameIntervalMs;
3384 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3385 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003386 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003387 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003388 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3390 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3391 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3392
Jonathan Yubc771b72017-12-08 17:04:29 -08003393 // Trigger quality adapt down, expect no change (min resolution reached).
3394 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003395 timestamp_ms += kFrameIntervalMs;
3396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3397 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003398 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3399 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3401 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3402 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3403
3404 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003405 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003406 timestamp_ms += kFrameIntervalMs;
3407 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3408 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003409 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003410 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3411 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3412 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3413 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3414
3415 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003416 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003417 timestamp_ms += kFrameIntervalMs;
3418 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3419 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003420 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3421 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3423 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3424 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3425
3426 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003427 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003428 timestamp_ms += kFrameIntervalMs;
3429 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3430 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003431 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003432 last_wants = source.sink_wants();
3433 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003435 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003436 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3437
3438 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003439 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003440 timestamp_ms += kFrameIntervalMs;
3441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3442 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003443 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003444 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3445 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003446 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003447 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3448
3449 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003450 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003451 timestamp_ms += kFrameIntervalMs;
3452 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003453 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003454 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003455 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003456 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003458 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003459 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003460
mflodmancc3d4422017-08-03 08:27:51 -07003461 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003462}
3463
mflodmancc3d4422017-08-03 08:27:51 -07003464TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003465 const int kWidth = 640;
3466 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003467
Erik Språng4c6ca302019-04-08 15:14:01 +02003468 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003469 DataRate::BitsPerSec(kTargetBitrateBps),
3470 DataRate::BitsPerSec(kTargetBitrateBps),
3471 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003472
perkj803d97f2016-11-01 11:45:46 -07003473 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003474 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003475 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003476 }
3477
mflodmancc3d4422017-08-03 08:27:51 -07003478 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003479 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003480 video_source_.IncomingCapturedFrame(CreateFrame(
3481 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003482 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003483 }
3484
mflodmancc3d4422017-08-03 08:27:51 -07003485 video_stream_encoder_->Stop();
3486 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003487 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003488
Ying Wangef3998f2019-12-09 13:06:53 +01003489 EXPECT_METRIC_EQ(
3490 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3491 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003492 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3493}
3494
mflodmancc3d4422017-08-03 08:27:51 -07003495TEST_F(VideoStreamEncoderTest,
3496 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003497 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003498 DataRate::BitsPerSec(kTargetBitrateBps),
3499 DataRate::BitsPerSec(kTargetBitrateBps),
3500 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003501 const int kWidth = 640;
3502 const int kHeight = 360;
3503
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003504 video_stream_encoder_->SetSource(&video_source_,
3505 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003506
3507 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3508 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003509 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003510 }
3511
mflodmancc3d4422017-08-03 08:27:51 -07003512 video_stream_encoder_->Stop();
3513 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003514 stats_proxy_.reset();
3515
3516 EXPECT_EQ(0,
3517 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3518}
3519
mflodmancc3d4422017-08-03 08:27:51 -07003520TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003521 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003522 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003523
3524 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003525 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003526 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003527 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3528 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003529
sprang57c2fff2017-01-16 06:24:02 -08003530 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003531 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003532 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003533 DataRate::BitsPerSec(kLowTargetBitrateBps),
3534 DataRate::BitsPerSec(kLowTargetBitrateBps),
3535 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003536
sprang57c2fff2017-01-16 06:24:02 -08003537 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003538 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3539 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003540 VideoBitrateAllocation bitrate_allocation =
3541 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003542 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003543 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003544 // TODO(srte): The use of millisecs here looks like an error, but the tests
3545 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003546 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003547
3548 // Not called on second frame.
3549 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3550 .Times(0);
3551 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003552 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3553 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003554 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003555
3556 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003557 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3558 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003559 const int64_t start_time_ms = rtc::TimeMillis();
3560 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3561 video_source_.IncomingCapturedFrame(
3562 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3563 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003564 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003565 }
3566
3567 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003568 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003569
mflodmancc3d4422017-08-03 08:27:51 -07003570 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003571}
3572
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003573TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3574 // 2 TLs configured, temporal layers supported by encoder.
3575 const int kNumTemporalLayers = 2;
3576 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3577 fake_encoder_.SetTemporalLayersSupported(0, true);
3578
3579 // Bitrate allocated across temporal layers.
3580 const int kTl0Bps = kTargetBitrateBps *
3581 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003582 kNumTemporalLayers, /*temporal_id*/ 0,
3583 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003584 const int kTl1Bps = kTargetBitrateBps *
3585 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003586 kNumTemporalLayers, /*temporal_id*/ 1,
3587 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003588 VideoBitrateAllocation expected_bitrate;
3589 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3590 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3591
3592 VerifyAllocatedBitrate(expected_bitrate);
3593 video_stream_encoder_->Stop();
3594}
3595
3596TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3597 // 2 TLs configured, temporal layers not supported by encoder.
3598 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3599 fake_encoder_.SetTemporalLayersSupported(0, false);
3600
3601 // Temporal layers not supported by the encoder.
3602 // Total bitrate should be at ti:0.
3603 VideoBitrateAllocation expected_bitrate;
3604 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3605
3606 VerifyAllocatedBitrate(expected_bitrate);
3607 video_stream_encoder_->Stop();
3608}
3609
3610TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3611 // 2 TLs configured, temporal layers only supported for first stream.
3612 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3613 fake_encoder_.SetTemporalLayersSupported(0, true);
3614 fake_encoder_.SetTemporalLayersSupported(1, false);
3615
3616 const int kS0Bps = 150000;
3617 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003618 kS0Bps *
3619 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3620 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003621 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003622 kS0Bps *
3623 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3624 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003625 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3626 // Temporal layers not supported by si:1.
3627 VideoBitrateAllocation expected_bitrate;
3628 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3629 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3630 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3631
3632 VerifyAllocatedBitrate(expected_bitrate);
3633 video_stream_encoder_->Stop();
3634}
3635
Niels Möller7dc26b72017-12-06 10:27:48 +01003636TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3637 const int kFrameWidth = 1280;
3638 const int kFrameHeight = 720;
3639 const int kFramerate = 24;
3640
Erik Språng4c6ca302019-04-08 15:14:01 +02003641 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003642 DataRate::BitsPerSec(kTargetBitrateBps),
3643 DataRate::BitsPerSec(kTargetBitrateBps),
3644 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003645 test::FrameForwarder source;
3646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003647 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003648
3649 // Insert a single frame, triggering initial configuration.
3650 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3652
3653 EXPECT_EQ(
3654 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3655 kDefaultFramerate);
3656
3657 // Trigger reconfigure encoder (without resetting the entire instance).
3658 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003659 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003660 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3661 video_encoder_config.number_of_streams = 1;
3662 video_encoder_config.video_stream_factory =
3663 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3664 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003665 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3667
3668 // Detector should be updated with fps limit from codec config.
3669 EXPECT_EQ(
3670 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3671 kFramerate);
3672
3673 // Trigger overuse, max framerate should be reduced.
3674 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3675 stats.input_frame_rate = kFramerate;
3676 stats_proxy_->SetMockStats(stats);
3677 video_stream_encoder_->TriggerCpuOveruse();
3678 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3679 int adapted_framerate =
3680 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3681 EXPECT_LT(adapted_framerate, kFramerate);
3682
3683 // Trigger underuse, max framerate should go back to codec configured fps.
3684 // Set extra low fps, to make sure it's actually reset, not just incremented.
3685 stats = stats_proxy_->GetStats();
3686 stats.input_frame_rate = adapted_framerate / 2;
3687 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003688 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003689 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3690 EXPECT_EQ(
3691 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3692 kFramerate);
3693
3694 video_stream_encoder_->Stop();
3695}
3696
3697TEST_F(VideoStreamEncoderTest,
3698 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3699 const int kFrameWidth = 1280;
3700 const int kFrameHeight = 720;
3701 const int kLowFramerate = 15;
3702 const int kHighFramerate = 25;
3703
Erik Språng4c6ca302019-04-08 15:14:01 +02003704 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003705 DataRate::BitsPerSec(kTargetBitrateBps),
3706 DataRate::BitsPerSec(kTargetBitrateBps),
3707 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003708 test::FrameForwarder source;
3709 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003710 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003711
3712 // Trigger initial configuration.
3713 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003714 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003715 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3716 video_encoder_config.number_of_streams = 1;
3717 video_encoder_config.video_stream_factory =
3718 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3719 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3720 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003721 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003722 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3723
3724 EXPECT_EQ(
3725 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3726 kLowFramerate);
3727
3728 // Trigger overuse, max framerate should be reduced.
3729 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3730 stats.input_frame_rate = kLowFramerate;
3731 stats_proxy_->SetMockStats(stats);
3732 video_stream_encoder_->TriggerCpuOveruse();
3733 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3734 int adapted_framerate =
3735 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3736 EXPECT_LT(adapted_framerate, kLowFramerate);
3737
3738 // Reconfigure the encoder with a new (higher max framerate), max fps should
3739 // still respect the adaptation.
3740 video_encoder_config.video_stream_factory =
3741 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3742 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3743 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003744 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003745 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3746
3747 EXPECT_EQ(
3748 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3749 adapted_framerate);
3750
3751 // Trigger underuse, max framerate should go back to codec configured fps.
3752 stats = stats_proxy_->GetStats();
3753 stats.input_frame_rate = adapted_framerate;
3754 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003755 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003756 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3757 EXPECT_EQ(
3758 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3759 kHighFramerate);
3760
3761 video_stream_encoder_->Stop();
3762}
3763
mflodmancc3d4422017-08-03 08:27:51 -07003764TEST_F(VideoStreamEncoderTest,
3765 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003766 const int kFrameWidth = 1280;
3767 const int kFrameHeight = 720;
3768 const int kFramerate = 24;
3769
Erik Språng4c6ca302019-04-08 15:14:01 +02003770 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003771 DataRate::BitsPerSec(kTargetBitrateBps),
3772 DataRate::BitsPerSec(kTargetBitrateBps),
3773 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003774 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003775 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003776 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003777
3778 // Trigger initial configuration.
3779 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003780 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003781 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3782 video_encoder_config.number_of_streams = 1;
3783 video_encoder_config.video_stream_factory =
3784 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3785 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003786 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003787 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003788 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003789
Niels Möller7dc26b72017-12-06 10:27:48 +01003790 EXPECT_EQ(
3791 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3792 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003793
3794 // Trigger overuse, max framerate should be reduced.
3795 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3796 stats.input_frame_rate = kFramerate;
3797 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003798 video_stream_encoder_->TriggerCpuOveruse();
3799 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003800 int adapted_framerate =
3801 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003802 EXPECT_LT(adapted_framerate, kFramerate);
3803
3804 // Change degradation preference to not enable framerate scaling. Target
3805 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003806 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003807 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003808 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003809 EXPECT_EQ(
3810 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3811 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003812
mflodmancc3d4422017-08-03 08:27:51 -07003813 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003814}
3815
mflodmancc3d4422017-08-03 08:27:51 -07003816TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003817 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003818 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003819 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3820 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3821 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003822 const int kWidth = 640;
3823 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003824
asaperssonfab67072017-04-04 05:51:49 -07003825 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003826
3827 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003828 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003829
3830 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003831 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003832
sprangc5d62e22017-04-02 23:53:04 -07003833 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003834
asaperssonfab67072017-04-04 05:51:49 -07003835 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003836 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003837 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003838
3839 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003840 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003841
sprangc5d62e22017-04-02 23:53:04 -07003842 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003843
mflodmancc3d4422017-08-03 08:27:51 -07003844 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003845}
3846
mflodmancc3d4422017-08-03 08:27:51 -07003847TEST_F(VideoStreamEncoderTest,
3848 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003849 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003850 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003851 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3852 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3853 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003854 const int kWidth = 640;
3855 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003856
3857 // We expect the n initial frames to get dropped.
3858 int i;
3859 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003860 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003861 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003862 }
3863 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003864 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003865 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003866
3867 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003868 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003869
mflodmancc3d4422017-08-03 08:27:51 -07003870 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003871}
3872
mflodmancc3d4422017-08-03 08:27:51 -07003873TEST_F(VideoStreamEncoderTest,
3874 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003875 const int kWidth = 640;
3876 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003877 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003878 DataRate::BitsPerSec(kLowTargetBitrateBps),
3879 DataRate::BitsPerSec(kLowTargetBitrateBps),
3880 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003881
3882 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003883 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003884 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003885
asaperssonfab67072017-04-04 05:51:49 -07003886 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003887 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003888 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003889
mflodmancc3d4422017-08-03 08:27:51 -07003890 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003891}
3892
mflodmancc3d4422017-08-03 08:27:51 -07003893TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003894 const int kWidth = 640;
3895 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003896 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003897
3898 VideoEncoderConfig video_encoder_config;
3899 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3900 // Make format different, to force recreation of encoder.
3901 video_encoder_config.video_format.parameters["foo"] = "foo";
3902 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003903 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003904 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003905 DataRate::BitsPerSec(kLowTargetBitrateBps),
3906 DataRate::BitsPerSec(kLowTargetBitrateBps),
3907 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003908
kthelgasonb83797b2017-02-14 11:57:25 -08003909 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003910 video_stream_encoder_->SetSource(&video_source_,
3911 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003912
asaperssonfab67072017-04-04 05:51:49 -07003913 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003914 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003915 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003916
mflodmancc3d4422017-08-03 08:27:51 -07003917 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003918 fake_encoder_.SetQualityScaling(true);
3919}
3920
Åsa Persson139f4dc2019-08-02 09:29:58 +02003921TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3922 webrtc::test::ScopedFieldTrials field_trials(
3923 "WebRTC-Video-QualityScalerSettings/"
3924 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3925 // Reset encoder for field trials to take effect.
3926 ConfigureEncoder(video_encoder_config_.Copy());
3927 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3928 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3929 const int kWidth = 640;
3930 const int kHeight = 360;
3931
3932 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003933 DataRate::BitsPerSec(kTargetBitrateBps),
3934 DataRate::BitsPerSec(kTargetBitrateBps),
3935 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003936 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3937 // Frame should not be dropped.
3938 WaitForEncodedFrame(1);
3939
3940 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003941 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3942 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3943 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003944 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3945 // Frame should not be dropped.
3946 WaitForEncodedFrame(2);
3947
3948 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003949 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3950 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3951 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003952 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3953 // Expect to drop this frame, the wait should time out.
3954 ExpectDroppedFrame();
3955
3956 // Expect the sink_wants to specify a scaled frame.
3957 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3958 video_stream_encoder_->Stop();
3959}
3960
Åsa Perssone644a032019-11-08 15:56:00 +01003961TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3962 webrtc::test::ScopedFieldTrials field_trials(
3963 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3964
3965 // Reset encoder for field trials to take effect.
3966 VideoEncoderConfig config = video_encoder_config_.Copy();
3967 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02003968 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01003969 ConfigureEncoder(std::move(config));
3970 fake_encoder_.SetQp(kQpLow);
3971
3972 // Enable MAINTAIN_FRAMERATE preference.
3973 AdaptingFrameForwarder source;
3974 source.set_adaptation_enabled(true);
3975 video_stream_encoder_->SetSource(&source,
3976 DegradationPreference::MAINTAIN_FRAMERATE);
3977
3978 // Start at low bitrate.
3979 const int kLowBitrateBps = 200000;
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003980 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(kLowBitrateBps),
3981 DataRate::BitsPerSec(kLowBitrateBps),
3982 DataRate::BitsPerSec(kLowBitrateBps),
3983 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003984
3985 // Expect first frame to be dropped and resolution to be limited.
3986 const int kWidth = 1280;
3987 const int kHeight = 720;
3988 const int64_t kFrameIntervalMs = 100;
3989 int64_t timestamp_ms = kFrameIntervalMs;
3990 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3991 ExpectDroppedFrame();
3992 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3993
3994 // Increase bitrate to encoder max.
Evan Shrubsoledff79252020-04-16 11:34:32 +02003995 video_stream_encoder_->OnBitrateUpdated(max_bitrate, max_bitrate, max_bitrate,
3996 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003997
3998 // Insert frames and advance |min_duration_ms|.
3999 for (size_t i = 1; i <= 10; i++) {
4000 timestamp_ms += kFrameIntervalMs;
4001 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4002 WaitForEncodedFrame(timestamp_ms);
4003 }
4004 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4005 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4006
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004007 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004008
4009 // Insert frame should trigger high BW and release quality limitation.
4010 timestamp_ms += kFrameIntervalMs;
4011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4012 WaitForEncodedFrame(timestamp_ms);
4013 VerifyFpsMaxResolutionMax(source.sink_wants());
4014
4015 // Frame should not be adapted.
4016 timestamp_ms += kFrameIntervalMs;
4017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4018 WaitForEncodedFrame(kWidth, kHeight);
4019 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4020
4021 video_stream_encoder_->Stop();
4022}
4023
mflodmancc3d4422017-08-03 08:27:51 -07004024TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004025 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4026 const int kTooSmallWidth = 10;
4027 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02004028 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004029 DataRate::BitsPerSec(kTargetBitrateBps),
4030 DataRate::BitsPerSec(kTargetBitrateBps),
4031 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004032
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004033 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004034 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004036 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004037 VerifyNoLimitation(source.sink_wants());
4038 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4039
4040 // Trigger adapt down, too small frame, expect no change.
4041 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004042 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004043 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004044 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07004045 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4046 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4047
mflodmancc3d4422017-08-03 08:27:51 -07004048 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004049}
4050
mflodmancc3d4422017-08-03 08:27:51 -07004051TEST_F(VideoStreamEncoderTest,
4052 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004053 const int kTooSmallWidth = 10;
4054 const int kTooSmallHeight = 10;
4055 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02004056 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004057 DataRate::BitsPerSec(kTargetBitrateBps),
4058 DataRate::BitsPerSec(kTargetBitrateBps),
4059 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004060
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004061 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004062 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004063 video_stream_encoder_->SetSource(&source,
4064 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004065 VerifyNoLimitation(source.sink_wants());
4066 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4067 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4068
4069 // Trigger adapt down, expect limited framerate.
4070 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004071 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004072 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004073 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4074 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4075 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4076 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4077
4078 // Trigger adapt down, too small frame, expect no change.
4079 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004080 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004081 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004082 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4083 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4084 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4085 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4086
mflodmancc3d4422017-08-03 08:27:51 -07004087 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004088}
4089
mflodmancc3d4422017-08-03 08:27:51 -07004090TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004091 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02004092 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004093 DataRate::BitsPerSec(kTargetBitrateBps),
4094 DataRate::BitsPerSec(kTargetBitrateBps),
4095 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004096 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004097 const int kFrameWidth = 1280;
4098 const int kFrameHeight = 720;
4099 video_source_.IncomingCapturedFrame(
4100 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004101 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004102 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004103}
4104
sprangb1ca0732017-02-01 08:38:12 -08004105// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004106TEST_F(VideoStreamEncoderTest,
4107 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004108 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004109 DataRate::BitsPerSec(kTargetBitrateBps),
4110 DataRate::BitsPerSec(kTargetBitrateBps),
4111 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004112
4113 const int kFrameWidth = 1280;
4114 const int kFrameHeight = 720;
4115 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004116 // requested by
4117 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004118 video_source_.set_adaptation_enabled(true);
4119
4120 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004121 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004122 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004123
4124 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004125 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004126 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004127 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004128 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004129
asaperssonfab67072017-04-04 05:51:49 -07004130 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004131 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004132 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004133 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004134 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004135
mflodmancc3d4422017-08-03 08:27:51 -07004136 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004137}
sprangfe627f32017-03-29 08:24:59 -07004138
mflodmancc3d4422017-08-03 08:27:51 -07004139TEST_F(VideoStreamEncoderTest,
4140 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004141 const int kFrameWidth = 1280;
4142 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004143
Erik Språng4c6ca302019-04-08 15:14:01 +02004144 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004145 DataRate::BitsPerSec(kTargetBitrateBps),
4146 DataRate::BitsPerSec(kTargetBitrateBps),
4147 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004148 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004149 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004150 video_source_.set_adaptation_enabled(true);
4151
sprang4847ae62017-06-27 07:06:52 -07004152 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004153
4154 video_source_.IncomingCapturedFrame(
4155 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004156 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004157
4158 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004159 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004160
4161 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004162 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004163 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004164 video_source_.IncomingCapturedFrame(
4165 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004166 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004167 }
4168
4169 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004170 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004171 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004172 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004173 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004174 video_source_.IncomingCapturedFrame(
4175 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004176 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004177 ++num_frames_dropped;
4178 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004179 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004180 }
4181 }
4182
sprang4847ae62017-06-27 07:06:52 -07004183 // Add some slack to account for frames dropped by the frame dropper.
4184 const int kErrorMargin = 1;
4185 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004186 kErrorMargin);
4187
4188 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004189 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004190 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004191 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004192 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004193 video_source_.IncomingCapturedFrame(
4194 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004195 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004196 ++num_frames_dropped;
4197 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004198 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004199 }
4200 }
sprang4847ae62017-06-27 07:06:52 -07004201 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004202 kErrorMargin);
4203
4204 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004205 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004206 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004207 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004208 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004209 video_source_.IncomingCapturedFrame(
4210 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004211 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004212 ++num_frames_dropped;
4213 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004214 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004215 }
4216 }
sprang4847ae62017-06-27 07:06:52 -07004217 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004218 kErrorMargin);
4219
4220 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004221 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004222 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004223 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004224 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004225 video_source_.IncomingCapturedFrame(
4226 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004227 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004228 ++num_frames_dropped;
4229 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004230 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004231 }
4232 }
4233 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4234
mflodmancc3d4422017-08-03 08:27:51 -07004235 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004236}
4237
mflodmancc3d4422017-08-03 08:27:51 -07004238TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004239 const int kFramerateFps = 5;
4240 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004241 const int kFrameWidth = 1280;
4242 const int kFrameHeight = 720;
4243
sprang4847ae62017-06-27 07:06:52 -07004244 // Reconfigure encoder with two temporal layers and screensharing, which will
4245 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004246 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004247
Erik Språng4c6ca302019-04-08 15:14:01 +02004248 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004249 DataRate::BitsPerSec(kTargetBitrateBps),
4250 DataRate::BitsPerSec(kTargetBitrateBps),
4251 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004252 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004253 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004254 video_source_.set_adaptation_enabled(true);
4255
sprang4847ae62017-06-27 07:06:52 -07004256 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004257
4258 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004259 rtc::VideoSinkWants last_wants;
4260 do {
4261 last_wants = video_source_.sink_wants();
4262
sprangc5d62e22017-04-02 23:53:04 -07004263 // Insert frames to get a new fps estimate...
4264 for (int j = 0; j < kFramerateFps; ++j) {
4265 video_source_.IncomingCapturedFrame(
4266 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004267 if (video_source_.last_sent_width()) {
4268 sink_.WaitForEncodedFrame(timestamp_ms);
4269 }
sprangc5d62e22017-04-02 23:53:04 -07004270 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004271 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004272 }
4273 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004274 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004275 } while (video_source_.sink_wants().max_framerate_fps <
4276 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004277
Jonathan Yubc771b72017-12-08 17:04:29 -08004278 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004279
mflodmancc3d4422017-08-03 08:27:51 -07004280 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004281}
asaperssonf7e294d2017-06-13 23:25:22 -07004282
mflodmancc3d4422017-08-03 08:27:51 -07004283TEST_F(VideoStreamEncoderTest,
4284 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004285 const int kWidth = 1280;
4286 const int kHeight = 720;
4287 const int64_t kFrameIntervalMs = 150;
4288 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004289 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004290 DataRate::BitsPerSec(kTargetBitrateBps),
4291 DataRate::BitsPerSec(kTargetBitrateBps),
4292 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004293
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004294 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004295 AdaptingFrameForwarder source;
4296 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004297 video_stream_encoder_->SetSource(&source,
4298 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004299 timestamp_ms += kFrameIntervalMs;
4300 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004301 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004302 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004303 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4304 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4305 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4306
4307 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004308 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004309 timestamp_ms += kFrameIntervalMs;
4310 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004311 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004312 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4313 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4315 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4316
4317 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004318 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004319 timestamp_ms += kFrameIntervalMs;
4320 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004321 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004322 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4323 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4324 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4325 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4326
4327 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004328 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004329 timestamp_ms += kFrameIntervalMs;
4330 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004331 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004332 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4333 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4334 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4335 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
4337 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004338 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004339 timestamp_ms += kFrameIntervalMs;
4340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004341 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004342 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4343 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4344 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4345 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4346
4347 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004348 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004349 timestamp_ms += kFrameIntervalMs;
4350 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004351 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004352 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4353 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4354 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4355 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
4357 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004358 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004359 timestamp_ms += kFrameIntervalMs;
4360 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004361 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004362 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4363 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4364 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4365 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4366
4367 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004368 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004369 timestamp_ms += kFrameIntervalMs;
4370 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004371 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004372 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4373 rtc::VideoSinkWants last_wants = source.sink_wants();
4374 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4375 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4376 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4377
4378 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004379 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004380 timestamp_ms += kFrameIntervalMs;
4381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004382 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004383 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4384 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4385 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4386 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4387
4388 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004389 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004390 timestamp_ms += kFrameIntervalMs;
4391 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004392 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004393 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4395 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4396 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4397
4398 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004399 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004400 timestamp_ms += kFrameIntervalMs;
4401 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004402 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004403 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4404 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4405 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4406 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4407
4408 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004409 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004410 timestamp_ms += kFrameIntervalMs;
4411 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004412 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004413 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4416 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4417
4418 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004419 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004420 timestamp_ms += kFrameIntervalMs;
4421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004422 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004423 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4426 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4427
4428 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004429 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004430 timestamp_ms += kFrameIntervalMs;
4431 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004432 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004433 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4435 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4436 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4437
4438 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004439 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004440 timestamp_ms += kFrameIntervalMs;
4441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004442 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004443 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4444 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4445 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4446 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4447
Åsa Persson30ab0152019-08-27 12:22:33 +02004448 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004449 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004450 timestamp_ms += kFrameIntervalMs;
4451 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004452 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004453 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004454 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4456 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4457 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4458
4459 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004460 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004461 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004462 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4463
mflodmancc3d4422017-08-03 08:27:51 -07004464 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004465}
4466
mflodmancc3d4422017-08-03 08:27:51 -07004467TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004468 const int kWidth = 1280;
4469 const int kHeight = 720;
4470 const int64_t kFrameIntervalMs = 150;
4471 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004472 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004473 DataRate::BitsPerSec(kTargetBitrateBps),
4474 DataRate::BitsPerSec(kTargetBitrateBps),
4475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004476
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004477 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004478 AdaptingFrameForwarder source;
4479 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004480 video_stream_encoder_->SetSource(&source,
4481 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004482 timestamp_ms += kFrameIntervalMs;
4483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004484 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004485 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4488 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4489 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4490 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4491 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4492
4493 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004494 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004495 timestamp_ms += kFrameIntervalMs;
4496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004497 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004498 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4501 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4502 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4503 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4504 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4505
4506 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004507 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004508 timestamp_ms += kFrameIntervalMs;
4509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004510 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004511 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4512 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4514 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4515 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4516 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4517 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4518
4519 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004520 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004521 timestamp_ms += kFrameIntervalMs;
4522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004523 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004524 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4527 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4528 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4529 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4530 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4531
4532 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004533 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004534 timestamp_ms += kFrameIntervalMs;
4535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004536 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004537 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4538 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4539 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4540 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4541 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4542 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4543 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4544
4545 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004546 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004547 timestamp_ms += kFrameIntervalMs;
4548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004549 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004550 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4552 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4553 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4554 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4555 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4556 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4557
4558 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004559 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004560 timestamp_ms += kFrameIntervalMs;
4561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004562 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004563 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004564 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4567 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4568 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4569 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4570 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4571
4572 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004573 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004574 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004575 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4576 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4577
mflodmancc3d4422017-08-03 08:27:51 -07004578 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004579}
4580
mflodmancc3d4422017-08-03 08:27:51 -07004581TEST_F(VideoStreamEncoderTest,
4582 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004583 const int kWidth = 640;
4584 const int kHeight = 360;
4585 const int kFpsLimit = 15;
4586 const int64_t kFrameIntervalMs = 150;
4587 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004588 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004589 DataRate::BitsPerSec(kTargetBitrateBps),
4590 DataRate::BitsPerSec(kTargetBitrateBps),
4591 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004592
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004593 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004594 AdaptingFrameForwarder source;
4595 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004596 video_stream_encoder_->SetSource(&source,
4597 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004598 timestamp_ms += kFrameIntervalMs;
4599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004600 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004601 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004602 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4603 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4604 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4605 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4606 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4607 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4608
4609 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004610 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004611 timestamp_ms += kFrameIntervalMs;
4612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004613 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004614 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4615 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4616 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4617 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4618 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4619 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4620 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4621
4622 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004623 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004624 timestamp_ms += kFrameIntervalMs;
4625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004626 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004627 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4629 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4630 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4631 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4632 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4633 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4634
4635 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004636 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004637 timestamp_ms += kFrameIntervalMs;
4638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004639 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004640 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4643 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4644 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4645 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4646 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4647
4648 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004649 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004650 timestamp_ms += kFrameIntervalMs;
4651 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004652 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004653 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4655 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4656 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4657 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4658 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4659 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4660
4661 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004662 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004663 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004664 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4665 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4666
mflodmancc3d4422017-08-03 08:27:51 -07004667 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004668}
4669
mflodmancc3d4422017-08-03 08:27:51 -07004670TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004671 const int kFrameWidth = 1920;
4672 const int kFrameHeight = 1080;
4673 // 3/4 of 1920.
4674 const int kAdaptedFrameWidth = 1440;
4675 // 3/4 of 1080 rounded down to multiple of 4.
4676 const int kAdaptedFrameHeight = 808;
4677 const int kFramerate = 24;
4678
Erik Språng4c6ca302019-04-08 15:14:01 +02004679 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004680 DataRate::BitsPerSec(kTargetBitrateBps),
4681 DataRate::BitsPerSec(kTargetBitrateBps),
4682 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004683 // Trigger reconfigure encoder (without resetting the entire instance).
4684 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004685 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004686 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4687 video_encoder_config.number_of_streams = 1;
4688 video_encoder_config.video_stream_factory =
4689 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004690 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004691 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004692 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004693
4694 video_source_.set_adaptation_enabled(true);
4695
4696 video_source_.IncomingCapturedFrame(
4697 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004698 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004699
4700 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004701 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004702 video_source_.IncomingCapturedFrame(
4703 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004704 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004705
mflodmancc3d4422017-08-03 08:27:51 -07004706 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004707}
4708
mflodmancc3d4422017-08-03 08:27:51 -07004709TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004710 const int kFrameWidth = 1280;
4711 const int kFrameHeight = 720;
4712 const int kLowFps = 2;
4713 const int kHighFps = 30;
4714
Erik Språng4c6ca302019-04-08 15:14:01 +02004715 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004716 DataRate::BitsPerSec(kTargetBitrateBps),
4717 DataRate::BitsPerSec(kTargetBitrateBps),
4718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004719
4720 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4721 max_framerate_ = kLowFps;
4722
4723 // Insert 2 seconds of 2fps video.
4724 for (int i = 0; i < kLowFps * 2; ++i) {
4725 video_source_.IncomingCapturedFrame(
4726 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4727 WaitForEncodedFrame(timestamp_ms);
4728 timestamp_ms += 1000 / kLowFps;
4729 }
4730
4731 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004732 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004733 DataRate::BitsPerSec(kTargetBitrateBps),
4734 DataRate::BitsPerSec(kTargetBitrateBps),
4735 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004736 video_source_.IncomingCapturedFrame(
4737 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4738 WaitForEncodedFrame(timestamp_ms);
4739 timestamp_ms += 1000 / kLowFps;
4740
4741 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4742
4743 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004744 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004745 const int kFrameIntervalMs = 1000 / kHighFps;
4746 max_framerate_ = kHighFps;
4747 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4748 video_source_.IncomingCapturedFrame(
4749 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4750 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4751 // be dropped if the encoder hans't been updated with the new higher target
4752 // framerate yet, causing it to overshoot the target bitrate and then
4753 // suffering the wrath of the media optimizer.
4754 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4755 timestamp_ms += kFrameIntervalMs;
4756 }
4757
4758 // Don expect correct measurement just yet, but it should be higher than
4759 // before.
4760 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4761
mflodmancc3d4422017-08-03 08:27:51 -07004762 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004763}
4764
mflodmancc3d4422017-08-03 08:27:51 -07004765TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004766 const int kFrameWidth = 1280;
4767 const int kFrameHeight = 720;
4768 const int kTargetBitrateBps = 1000000;
4769
4770 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004771 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004772 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004773 DataRate::BitsPerSec(kTargetBitrateBps),
4774 DataRate::BitsPerSec(kTargetBitrateBps),
4775 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004777
4778 // Insert a first video frame, causes another bitrate update.
4779 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4780 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4781 video_source_.IncomingCapturedFrame(
4782 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4783 WaitForEncodedFrame(timestamp_ms);
4784
4785 // Next, simulate video suspension due to pacer queue overrun.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004786 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
4787 DataRate::BitsPerSec(0),
4788 DataRate::BitsPerSec(0), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004789
4790 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004791 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004792 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004793
4794 // Bitrate observer should not be called.
4795 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4796 video_source_.IncomingCapturedFrame(
4797 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4798 ExpectDroppedFrame();
4799
mflodmancc3d4422017-08-03 08:27:51 -07004800 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004801}
ilnik6b826ef2017-06-16 06:53:48 -07004802
Niels Möller4db138e2018-04-19 09:04:13 +02004803TEST_F(VideoStreamEncoderTest,
4804 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4805 const int kFrameWidth = 1280;
4806 const int kFrameHeight = 720;
4807 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004808 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004809 DataRate::BitsPerSec(kTargetBitrateBps),
4810 DataRate::BitsPerSec(kTargetBitrateBps),
4811 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004812 video_source_.IncomingCapturedFrame(
4813 CreateFrame(1, kFrameWidth, kFrameHeight));
4814 WaitForEncodedFrame(1);
4815 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4816 .low_encode_usage_threshold_percent,
4817 default_options.low_encode_usage_threshold_percent);
4818 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4819 .high_encode_usage_threshold_percent,
4820 default_options.high_encode_usage_threshold_percent);
4821 video_stream_encoder_->Stop();
4822}
4823
4824TEST_F(VideoStreamEncoderTest,
4825 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4826 const int kFrameWidth = 1280;
4827 const int kFrameHeight = 720;
4828 CpuOveruseOptions hardware_options;
4829 hardware_options.low_encode_usage_threshold_percent = 150;
4830 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004831 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004832
Erik Språng4c6ca302019-04-08 15:14:01 +02004833 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004834 DataRate::BitsPerSec(kTargetBitrateBps),
4835 DataRate::BitsPerSec(kTargetBitrateBps),
4836 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004837 video_source_.IncomingCapturedFrame(
4838 CreateFrame(1, kFrameWidth, kFrameHeight));
4839 WaitForEncodedFrame(1);
4840 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4841 .low_encode_usage_threshold_percent,
4842 hardware_options.low_encode_usage_threshold_percent);
4843 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4844 .high_encode_usage_threshold_percent,
4845 hardware_options.high_encode_usage_threshold_percent);
4846 video_stream_encoder_->Stop();
4847}
4848
Niels Möller6bb5ab92019-01-11 11:11:10 +01004849TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4850 const int kFrameWidth = 320;
4851 const int kFrameHeight = 240;
4852 const int kFps = 30;
4853 const int kTargetBitrateBps = 120000;
4854 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4855
Erik Språng4c6ca302019-04-08 15:14:01 +02004856 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004857 DataRate::BitsPerSec(kTargetBitrateBps),
4858 DataRate::BitsPerSec(kTargetBitrateBps),
4859 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004860
4861 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4862 max_framerate_ = kFps;
4863
4864 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4865 fake_encoder_.SimulateOvershoot(1.0);
4866 int num_dropped = 0;
4867 for (int i = 0; i < kNumFramesInRun; ++i) {
4868 video_source_.IncomingCapturedFrame(
4869 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4870 // Wait up to two frame durations for a frame to arrive.
4871 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4872 ++num_dropped;
4873 }
4874 timestamp_ms += 1000 / kFps;
4875 }
4876
Erik Språnga8d48ab2019-02-08 14:17:40 +01004877 // Framerate should be measured to be near the expected target rate.
4878 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4879
4880 // Frame drops should be within 5% of expected 0%.
4881 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004882
4883 // Make encoder produce frames at double the expected bitrate during 3 seconds
4884 // of video, verify number of drops. Rate needs to be slightly changed in
4885 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004886 double overshoot_factor = 2.0;
4887 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4888 // With bitrate adjuster, when need to overshoot even more to trigger
4889 // frame dropping.
4890 overshoot_factor *= 2;
4891 }
4892 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004893 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004894 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4895 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4896 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004897 num_dropped = 0;
4898 for (int i = 0; i < kNumFramesInRun; ++i) {
4899 video_source_.IncomingCapturedFrame(
4900 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4901 // Wait up to two frame durations for a frame to arrive.
4902 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4903 ++num_dropped;
4904 }
4905 timestamp_ms += 1000 / kFps;
4906 }
4907
Erik Språng4c6ca302019-04-08 15:14:01 +02004908 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004909 DataRate::BitsPerSec(kTargetBitrateBps),
4910 DataRate::BitsPerSec(kTargetBitrateBps),
4911 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004912
4913 // Target framerate should be still be near the expected target, despite
4914 // the frame drops.
4915 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4916
4917 // Frame drops should be within 5% of expected 50%.
4918 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004919
4920 video_stream_encoder_->Stop();
4921}
4922
4923TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4924 const int kFrameWidth = 320;
4925 const int kFrameHeight = 240;
4926 const int kActualInputFps = 24;
4927 const int kTargetBitrateBps = 120000;
4928
4929 ASSERT_GT(max_framerate_, kActualInputFps);
4930
4931 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4932 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004933 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004934 DataRate::BitsPerSec(kTargetBitrateBps),
4935 DataRate::BitsPerSec(kTargetBitrateBps),
4936 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004937
4938 // Insert 3 seconds of video, with an input fps lower than configured max.
4939 for (int i = 0; i < kActualInputFps * 3; ++i) {
4940 video_source_.IncomingCapturedFrame(
4941 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4942 // Wait up to two frame durations for a frame to arrive.
4943 WaitForEncodedFrame(timestamp_ms);
4944 timestamp_ms += 1000 / kActualInputFps;
4945 }
4946
4947 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4948
4949 video_stream_encoder_->Stop();
4950}
4951
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004952TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4953 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004954 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004955 DataRate::BitsPerSec(kTargetBitrateBps),
4956 DataRate::BitsPerSec(kTargetBitrateBps),
4957 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004958
4959 fake_encoder_.BlockNextEncode();
4960 video_source_.IncomingCapturedFrame(
4961 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4962 WaitForEncodedFrame(1);
4963 // On the very first frame full update should be forced.
4964 rect = fake_encoder_.GetLastUpdateRect();
4965 EXPECT_EQ(rect.offset_x, 0);
4966 EXPECT_EQ(rect.offset_y, 0);
4967 EXPECT_EQ(rect.height, codec_height_);
4968 EXPECT_EQ(rect.width, codec_width_);
4969 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4970 // call to ContinueEncode.
4971 video_source_.IncomingCapturedFrame(
4972 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4973 ExpectDroppedFrame();
4974 video_source_.IncomingCapturedFrame(
4975 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4976 ExpectDroppedFrame();
4977 fake_encoder_.ContinueEncode();
4978 WaitForEncodedFrame(3);
4979 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4980 rect = fake_encoder_.GetLastUpdateRect();
4981 EXPECT_EQ(rect.offset_x, 1);
4982 EXPECT_EQ(rect.offset_y, 0);
4983 EXPECT_EQ(rect.width, 10);
4984 EXPECT_EQ(rect.height, 1);
4985
4986 video_source_.IncomingCapturedFrame(
4987 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4988 WaitForEncodedFrame(4);
4989 // Previous frame was encoded, so no accumulation should happen.
4990 rect = fake_encoder_.GetLastUpdateRect();
4991 EXPECT_EQ(rect.offset_x, 0);
4992 EXPECT_EQ(rect.offset_y, 0);
4993 EXPECT_EQ(rect.width, 1);
4994 EXPECT_EQ(rect.height, 1);
4995
4996 video_stream_encoder_->Stop();
4997}
4998
Erik Språngd7329ca2019-02-21 21:19:53 +01004999TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02005000 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005001 DataRate::BitsPerSec(kTargetBitrateBps),
5002 DataRate::BitsPerSec(kTargetBitrateBps),
5003 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005004
5005 // First frame is always keyframe.
5006 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5007 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005008 EXPECT_THAT(
5009 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005010 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005011
5012 // Insert delta frame.
5013 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5014 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005015 EXPECT_THAT(
5016 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005017 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005018
5019 // Request next frame be a key-frame.
5020 video_stream_encoder_->SendKeyFrame();
5021 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5022 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005023 EXPECT_THAT(
5024 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005025 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005026
5027 video_stream_encoder_->Stop();
5028}
5029
5030TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5031 // Setup simulcast with three streams.
5032 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01005033 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005034 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5035 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5036 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005037 // Wait for all three layers before triggering event.
5038 sink_.SetNumExpectedLayers(3);
5039
5040 // First frame is always keyframe.
5041 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5042 WaitForEncodedFrame(1);
5043 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005044 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5045 VideoFrameType::kVideoFrameKey,
5046 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005047
5048 // Insert delta frame.
5049 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5050 WaitForEncodedFrame(2);
5051 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005052 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5053 VideoFrameType::kVideoFrameDelta,
5054 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005055
5056 // Request next frame be a key-frame.
5057 // Only first stream is configured to produce key-frame.
5058 video_stream_encoder_->SendKeyFrame();
5059 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5060 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005061
5062 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5063 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005064 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005065 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005066 VideoFrameType::kVideoFrameKey,
5067 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005068
5069 video_stream_encoder_->Stop();
5070}
5071
5072TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5073 // Configure internal source factory and setup test again.
5074 encoder_factory_.SetHasInternalSource(true);
5075 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005076 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005077 DataRate::BitsPerSec(kTargetBitrateBps),
5078 DataRate::BitsPerSec(kTargetBitrateBps),
5079 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005080
5081 // Call encoder directly, simulating internal source where encoded frame
5082 // callback in VideoStreamEncoder is called despite no OnFrame().
5083 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5084 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005085 EXPECT_THAT(
5086 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005087 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005088
Niels Möller8f7ce222019-03-21 15:43:58 +01005089 const std::vector<VideoFrameType> kDeltaFrame = {
5090 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005091 // Need to set timestamp manually since manually for injected frame.
5092 VideoFrame frame = CreateFrame(101, nullptr);
5093 frame.set_timestamp(101);
5094 fake_encoder_.InjectFrame(frame, false);
5095 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005096 EXPECT_THAT(
5097 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005098 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005099
5100 // Request key-frame. The forces a dummy frame down into the encoder.
5101 fake_encoder_.ExpectNullFrame();
5102 video_stream_encoder_->SendKeyFrame();
5103 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005104 EXPECT_THAT(
5105 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005106 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005107
5108 video_stream_encoder_->Stop();
5109}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005110
5111TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5112 // Configure internal source factory and setup test again.
5113 encoder_factory_.SetHasInternalSource(true);
5114 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005115 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005116 DataRate::BitsPerSec(kTargetBitrateBps),
5117 DataRate::BitsPerSec(kTargetBitrateBps),
5118 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005119
5120 int64_t timestamp = 1;
5121 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005122 image.SetEncodedData(
5123 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005124 image.capture_time_ms_ = ++timestamp;
5125 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5126 const int64_t kEncodeFinishDelayMs = 10;
5127 image.timing_.encode_start_ms = timestamp;
5128 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5129 fake_encoder_.InjectEncodedImage(image);
5130 // Wait for frame without incrementing clock.
5131 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5132 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5133 // capture timestamp should be kEncodeFinishDelayMs in the past.
5134 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5135 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5136 kEncodeFinishDelayMs);
5137
5138 video_stream_encoder_->Stop();
5139}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005140
5141TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5142 // Configure internal source factory and setup test again.
5143 encoder_factory_.SetHasInternalSource(true);
5144 ResetEncoder("H264", 1, 1, 1, false);
5145
5146 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5147 image._frameType = VideoFrameType::kVideoFrameKey;
5148
5149 CodecSpecificInfo codec_specific_info;
5150 codec_specific_info.codecType = kVideoCodecH264;
5151
5152 RTPFragmentationHeader fragmentation;
5153 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5154 fragmentation.fragmentationOffset[0] = 4;
5155 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5156
5157 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5158 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5159
5160 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5161 testing::ElementsAreArray(optimal_sps));
5162 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5163 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5164 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5165 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5166
5167 video_stream_encoder_->Stop();
5168}
5169
5170TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5171 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5172 0x00, 0x00, 0x03, 0x03, 0xF4,
5173 0x05, 0x03, 0xC7, 0xC0};
5174
5175 // Configure internal source factory and setup test again.
5176 encoder_factory_.SetHasInternalSource(true);
5177 ResetEncoder("H264", 1, 1, 1, false);
5178
5179 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5180 image._frameType = VideoFrameType::kVideoFrameKey;
5181
5182 CodecSpecificInfo codec_specific_info;
5183 codec_specific_info.codecType = kVideoCodecH264;
5184
5185 RTPFragmentationHeader fragmentation;
5186 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5187 fragmentation.fragmentationOffset[0] = 4;
5188 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5189
5190 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5191 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5192
5193 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5194 testing::ElementsAreArray(optimal_sps));
5195 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5196 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5197 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5198 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5199
5200 video_stream_encoder_->Stop();
5201}
5202
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005203TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5204 const int kFrameWidth = 1280;
5205 const int kFrameHeight = 720;
5206 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5207
5208 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005209 DataRate::BitsPerSec(kTargetBitrateBps),
5210 DataRate::BitsPerSec(kTargetBitrateBps),
5211 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005212 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5213
5214 // Insert a first video frame. It should be dropped because of downscale in
5215 // resolution.
5216 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5217 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5218 frame.set_rotation(kVideoRotation_270);
5219 video_source_.IncomingCapturedFrame(frame);
5220
5221 ExpectDroppedFrame();
5222
5223 // Second frame is downscaled.
5224 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5225 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5226 frame.set_rotation(kVideoRotation_90);
5227 video_source_.IncomingCapturedFrame(frame);
5228
5229 WaitForEncodedFrame(timestamp_ms);
5230 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5231
5232 // Insert another frame, also downscaled.
5233 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5234 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5235 frame.set_rotation(kVideoRotation_180);
5236 video_source_.IncomingCapturedFrame(frame);
5237
5238 WaitForEncodedFrame(timestamp_ms);
5239 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5240
5241 video_stream_encoder_->Stop();
5242}
5243
Erik Språng5056af02019-09-02 15:53:11 +02005244TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5245 const int kFrameWidth = 320;
5246 const int kFrameHeight = 180;
5247
5248 // Initial rate.
5249 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005250 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5251 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5252 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005253 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005254 /*rtt_ms=*/0,
5255 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005256
5257 // Insert a first video frame so that encoder gets configured.
5258 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5259 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5260 frame.set_rotation(kVideoRotation_270);
5261 video_source_.IncomingCapturedFrame(frame);
5262 WaitForEncodedFrame(timestamp_ms);
5263
5264 // Set a target rate below the minimum allowed by the codec settings.
5265 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005266 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5267 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Erik Språng5056af02019-09-02 15:53:11 +02005268 video_stream_encoder_->OnBitrateUpdated(
5269 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005270 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005271 /*link_allocation=*/target_rate,
5272 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005273 /*rtt_ms=*/0,
5274 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005275 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5276
5277 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5278 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5279 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005280 DataRate allocation_sum =
5281 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005282 EXPECT_EQ(min_rate, allocation_sum);
5283 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5284
5285 video_stream_encoder_->Stop();
5286}
5287
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005288TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005289 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005290 DataRate::BitsPerSec(kTargetBitrateBps),
5291 DataRate::BitsPerSec(kTargetBitrateBps),
5292 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005293 // Capture a frame and wait for it to synchronize with the encoder thread.
5294 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5295 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5296 WaitForEncodedFrame(1);
5297
5298 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5299 ASSERT_TRUE(prev_rate_settings.has_value());
5300 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5301 kDefaultFramerate);
5302
5303 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5304 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5305 timestamp_ms += 1000 / kDefaultFramerate;
5306 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5307 WaitForEncodedFrame(timestamp_ms);
5308 }
5309 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5310 kDefaultFramerate);
5311 // Capture larger frame to trigger a reconfigure.
5312 codec_height_ *= 2;
5313 codec_width_ *= 2;
5314 timestamp_ms += 1000 / kDefaultFramerate;
5315 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5316 WaitForEncodedFrame(timestamp_ms);
5317
5318 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5319 auto current_rate_settings =
5320 fake_encoder_.GetAndResetLastRateControlSettings();
5321 // Ensure we have actually reconfigured twice
5322 // The rate settings should have been set again even though
5323 // they haven't changed.
5324 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005325 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005326
5327 video_stream_encoder_->Stop();
5328}
5329
philipeld9cc8c02019-09-16 14:53:40 +02005330struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5331 MOCK_METHOD0(RequestEncoderFallback, void());
5332 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 11:30:00 +01005333 MOCK_METHOD1(RequestEncoderSwitch,
5334 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 14:53:40 +02005335};
5336
5337TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5338 constexpr int kDontCare = 100;
5339
5340 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5341 video_send_config_.encoder_settings.encoder_switch_request_callback =
5342 &switch_callback;
5343 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5344 encoder_config.codec_type = kVideoCodecVP8;
5345 webrtc::test::ScopedFieldTrials field_trial(
5346 "WebRTC-NetworkCondition-EncoderSwitch/"
5347 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5348 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5349
5350 // Reset encoder for new configuration to take effect.
5351 ConfigureEncoder(std::move(encoder_config));
5352
5353 // Send one frame to trigger ReconfigureEncoder.
5354 video_source_.IncomingCapturedFrame(
5355 CreateFrame(kDontCare, kDontCare, kDontCare));
5356
5357 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005358 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5359 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005360 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005361 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005362
5363 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005364 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5365 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5366 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005367 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005368 /*rtt_ms=*/0,
5369 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005370
5371 video_stream_encoder_->Stop();
5372}
5373
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005374TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5375 constexpr int kDontCare = 100;
5376
5377 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5378 video_send_config_.encoder_settings.encoder_switch_request_callback =
5379 &switch_callback;
5380 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5381 encoder_config.codec_type = kVideoCodecVP8;
5382 webrtc::test::ScopedFieldTrials field_trial(
5383 "WebRTC-NetworkCondition-EncoderSwitch/"
5384 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5385 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5386
5387 // Reset encoder for new configuration to take effect.
5388 ConfigureEncoder(std::move(encoder_config));
5389
5390 // Send one frame to trigger ReconfigureEncoder.
5391 video_source_.IncomingCapturedFrame(
5392 CreateFrame(kDontCare, kDontCare, kDontCare));
5393
5394 using Config = EncoderSwitchRequestCallback::Config;
5395 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5396 .Times(0);
5397
5398 video_stream_encoder_->OnBitrateUpdated(
5399 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5400 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5401 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5402 /*fraction_lost=*/0,
5403 /*rtt_ms=*/0,
5404 /*cwnd_reduce_ratio=*/0);
5405
5406 video_stream_encoder_->Stop();
5407}
5408
philipeld9cc8c02019-09-16 14:53:40 +02005409TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5410 constexpr int kSufficientBitrateToNotDrop = 1000;
5411 constexpr int kHighRes = 500;
5412 constexpr int kLowRes = 100;
5413
5414 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5415 video_send_config_.encoder_settings.encoder_switch_request_callback =
5416 &switch_callback;
5417 webrtc::test::ScopedFieldTrials field_trial(
5418 "WebRTC-NetworkCondition-EncoderSwitch/"
5419 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5420 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5421 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5422 encoder_config.codec_type = kVideoCodecH264;
5423
5424 // Reset encoder for new configuration to take effect.
5425 ConfigureEncoder(std::move(encoder_config));
5426
5427 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5428 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5429 // not fail.
5430 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005431 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5432 /*stable_target_bitrate=*/
5433 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5434 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005435 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005436 /*rtt_ms=*/0,
5437 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005438
5439 // Send one frame to trigger ReconfigureEncoder.
5440 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5441 WaitForEncodedFrame(1);
5442
5443 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005444 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5445 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005446 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005447 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005448
5449 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5450 WaitForEncodedFrame(2);
5451
5452 video_stream_encoder_->Stop();
5453}
5454
philipel9b058032020-02-10 11:30:00 +01005455TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5456 constexpr int kDontCare = 100;
5457 StrictMock<MockEncoderSelector> encoder_selector;
5458 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5459 &fake_encoder_, &encoder_selector);
5460 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5461
5462 // Reset encoder for new configuration to take effect.
5463 ConfigureEncoder(video_encoder_config_.Copy());
5464
5465 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5466
5467 video_source_.IncomingCapturedFrame(
5468 CreateFrame(kDontCare, kDontCare, kDontCare));
5469 video_stream_encoder_->Stop();
5470
5471 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5472 // to it's factory, so in order for the encoder instance in the
5473 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5474 // reset the |video_stream_encoder_| here.
5475 video_stream_encoder_.reset();
5476}
5477
5478TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5479 constexpr int kDontCare = 100;
5480
5481 NiceMock<MockEncoderSelector> encoder_selector;
5482 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5483 video_send_config_.encoder_settings.encoder_switch_request_callback =
5484 &switch_callback;
5485 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5486 &fake_encoder_, &encoder_selector);
5487 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5488
5489 // Reset encoder for new configuration to take effect.
5490 ConfigureEncoder(video_encoder_config_.Copy());
5491
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005492 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005493 .WillByDefault(Return(SdpVideoFormat("AV1")));
5494 EXPECT_CALL(switch_callback,
5495 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5496 Field(&SdpVideoFormat::name, "AV1"))));
5497
5498 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005499 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5500 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5501 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005502 /*fraction_lost=*/0,
5503 /*rtt_ms=*/0,
5504 /*cwnd_reduce_ratio=*/0);
5505
5506 video_stream_encoder_->Stop();
5507}
5508
5509TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5510 constexpr int kSufficientBitrateToNotDrop = 1000;
5511 constexpr int kDontCare = 100;
5512
5513 NiceMock<MockVideoEncoder> video_encoder;
5514 NiceMock<MockEncoderSelector> encoder_selector;
5515 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5516 video_send_config_.encoder_settings.encoder_switch_request_callback =
5517 &switch_callback;
5518 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5519 &video_encoder, &encoder_selector);
5520 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5521
5522 // Reset encoder for new configuration to take effect.
5523 ConfigureEncoder(video_encoder_config_.Copy());
5524
5525 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5526 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5527 // not fail.
5528 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005529 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5530 /*stable_target_bitrate=*/
5531 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5532 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005533 /*fraction_lost=*/0,
5534 /*rtt_ms=*/0,
5535 /*cwnd_reduce_ratio=*/0);
5536
5537 ON_CALL(video_encoder, Encode(_, _))
5538 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5539 ON_CALL(encoder_selector, OnEncoderBroken())
5540 .WillByDefault(Return(SdpVideoFormat("AV2")));
5541
5542 rtc::Event encode_attempted;
5543 EXPECT_CALL(switch_callback,
5544 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5545 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5546 EXPECT_EQ(format.name, "AV2");
5547 encode_attempted.Set();
5548 });
5549
5550 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5551 encode_attempted.Wait(3000);
5552
5553 video_stream_encoder_->Stop();
5554
5555 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5556 // to it's factory, so in order for the encoder instance in the
5557 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5558 // reset the |video_stream_encoder_| here.
5559 video_stream_encoder_.reset();
5560}
5561
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005562TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005563 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005564 const int kFrameWidth = 320;
5565 const int kFrameHeight = 180;
5566
5567 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005568 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005569 video_stream_encoder_->OnBitrateUpdated(
5570 /*target_bitrate=*/rate,
5571 /*stable_target_bitrate=*/rate,
5572 /*link_allocation=*/rate,
5573 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005574 /*rtt_ms=*/0,
5575 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005576
5577 // Insert a first video frame so that encoder gets configured.
5578 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5579 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5580 frame.set_rotation(kVideoRotation_270);
5581 video_source_.IncomingCapturedFrame(frame);
5582 WaitForEncodedFrame(timestamp_ms);
5583 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5584
5585 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005586 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005587 video_stream_encoder_->OnBitrateUpdated(
5588 /*target_bitrate=*/new_stable_rate,
5589 /*stable_target_bitrate=*/new_stable_rate,
5590 /*link_allocation=*/rate,
5591 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005592 /*rtt_ms=*/0,
5593 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005594 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5595 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5596 video_stream_encoder_->Stop();
5597}
5598
5599TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005600 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005601 const int kFrameWidth = 320;
5602 const int kFrameHeight = 180;
5603
5604 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005605 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005606 video_stream_encoder_->OnBitrateUpdated(
5607 /*target_bitrate=*/rate,
5608 /*stable_target_bitrate=*/rate,
5609 /*link_allocation=*/rate,
5610 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005611 /*rtt_ms=*/0,
5612 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005613
5614 // Insert a first video frame so that encoder gets configured.
5615 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5616 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5617 frame.set_rotation(kVideoRotation_270);
5618 video_source_.IncomingCapturedFrame(frame);
5619 WaitForEncodedFrame(timestamp_ms);
5620 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5621
5622 // Set a higher target rate without changing the link_allocation. Should not
5623 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005624 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005625 video_stream_encoder_->OnBitrateUpdated(
5626 /*target_bitrate=*/rate,
5627 /*stable_target_bitrate=*/new_stable_rate,
5628 /*link_allocation=*/rate,
5629 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005630 /*rtt_ms=*/0,
5631 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005632 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5633 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5634 video_stream_encoder_->Stop();
5635}
5636
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005637TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5638 test::ScopedFieldTrials field_trials(
5639 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5640 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5641 const int kFramerateFps = 30;
5642 const int kWidth = 1920;
5643 const int kHeight = 1080;
5644 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5645 // Works on screenshare mode.
5646 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5647 // We rely on the automatic resolution adaptation, but we handle framerate
5648 // adaptation manually by mocking the stats proxy.
5649 video_source_.set_adaptation_enabled(true);
5650
5651 // BALANCED degradation preference is required for this feature.
5652 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005653 DataRate::BitsPerSec(kTargetBitrateBps),
5654 DataRate::BitsPerSec(kTargetBitrateBps),
5655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005656 video_stream_encoder_->SetSource(&video_source_,
5657 webrtc::DegradationPreference::BALANCED);
5658 VerifyNoLimitation(video_source_.sink_wants());
5659
5660 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5661 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5662
5663 // Pass enough frames with the full update to trigger animation detection.
5664 for (int i = 0; i < kNumFrames; ++i) {
5665 int64_t timestamp_ms =
5666 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5667 frame.set_ntp_time_ms(timestamp_ms);
5668 frame.set_timestamp_us(timestamp_ms * 1000);
5669 video_source_.IncomingCapturedFrame(frame);
5670 WaitForEncodedFrame(timestamp_ms);
5671 }
5672
5673 // Resolution should be limited.
5674 rtc::VideoSinkWants expected;
5675 expected.max_framerate_fps = kFramerateFps;
5676 expected.max_pixel_count = 1280 * 720 + 1;
5677 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5678
5679 // Pass one frame with no known update.
5680 // Resolution cap should be removed immediately.
5681 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5682 frame.set_ntp_time_ms(timestamp_ms);
5683 frame.set_timestamp_us(timestamp_ms * 1000);
5684 frame.clear_update_rect();
5685
5686 video_source_.IncomingCapturedFrame(frame);
5687 WaitForEncodedFrame(timestamp_ms);
5688
5689 // Resolution should be unlimited now.
5690 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5691
5692 video_stream_encoder_->Stop();
5693}
5694
perkj26091b12016-09-01 01:17:40 -07005695} // namespace webrtc