blob: 30660bbce6fc3cf99c114de1010551daab1dced3 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020040#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "system_wrappers/include/sleep.h"
42#include "test/encoder_settings.h"
43#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020044#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010045#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/gmock.h"
47#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020048#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020049#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070050
51namespace webrtc {
52
sprang57c2fff2017-01-16 06:24:02 -080053using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020054using ::testing::AllOf;
55using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010056using ::testing::Matcher;
57using ::testing::NiceMock;
58using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080060
perkj803d97f2016-11-01 11:45:46 -070061namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020062const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010063const int kQpLow = 1;
64const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int kMinFramerateFps = 2;
66const int kMinBalancedFramerateFps = 7;
67const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080068const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010069const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020070const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010071const uint32_t kSimulcastTargetBitrateBps = 3150000;
72const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080073const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070074const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020076const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020077const VideoEncoder::ResolutionBitrateLimits
78 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
79const VideoEncoder::ResolutionBitrateLimits
80 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080081
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020082uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
83 0x00, 0x00, 0x03, 0x03, 0xF4,
84 0x05, 0x03, 0xC7, 0xE0, 0x1B,
85 0x41, 0x10, 0x8D, 0x00};
86
perkj803d97f2016-11-01 11:45:46 -070087class TestBuffer : public webrtc::I420Buffer {
88 public:
89 TestBuffer(rtc::Event* event, int width, int height)
90 : I420Buffer(width, height), event_(event) {}
91
92 private:
93 friend class rtc::RefCountedObject<TestBuffer>;
94 ~TestBuffer() override {
95 if (event_)
96 event_->Set();
97 }
98 rtc::Event* const event_;
99};
100
Noah Richards51db4212019-06-12 06:59:12 -0700101// A fake native buffer that can't be converted to I420.
102class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
103 public:
104 FakeNativeBuffer(rtc::Event* event, int width, int height)
105 : event_(event), width_(width), height_(height) {}
106 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
107 int width() const override { return width_; }
108 int height() const override { return height_; }
109 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
110 return nullptr;
111 }
112
113 private:
114 friend class rtc::RefCountedObject<FakeNativeBuffer>;
115 ~FakeNativeBuffer() override {
116 if (event_)
117 event_->Set();
118 }
119 rtc::Event* const event_;
120 const int width_;
121 const int height_;
122};
123
Niels Möller7dc26b72017-12-06 10:27:48 +0100124class CpuOveruseDetectorProxy : public OveruseFrameDetector {
125 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200126 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
127 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100128 last_target_framerate_fps_(-1) {}
129 virtual ~CpuOveruseDetectorProxy() {}
130
131 void OnTargetFramerateUpdated(int framerate_fps) override {
132 rtc::CritScope cs(&lock_);
133 last_target_framerate_fps_ = framerate_fps;
134 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
135 }
136
137 int GetLastTargetFramerate() {
138 rtc::CritScope cs(&lock_);
139 return last_target_framerate_fps_;
140 }
141
Niels Möller4db138e2018-04-19 09:04:13 +0200142 CpuOveruseOptions GetOptions() { return options_; }
143
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 private:
145 rtc::CriticalSection lock_;
146 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
147};
148
Henrik Boström91aa7322020-04-28 12:24:33 +0200149class FakeQualityScalerQpUsageHandlerCallback
150 : public QualityScalerQpUsageHandlerCallbackInterface {
151 public:
152 FakeQualityScalerQpUsageHandlerCallback()
153 : QualityScalerQpUsageHandlerCallbackInterface() {}
154 ~FakeQualityScalerQpUsageHandlerCallback() override {}
155
156 void OnQpUsageHandled(bool clear_qp_samples) override {
157 clear_qp_samples_result_ = clear_qp_samples;
158 }
159
160 absl::optional<bool> clear_qp_samples_result() const {
161 return clear_qp_samples_result_;
162 }
163
164 private:
165 absl::optional<bool> clear_qp_samples_result_;
166};
167
mflodmancc3d4422017-08-03 08:27:51 -0700168class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700169 public:
Niels Möller213618e2018-07-24 09:29:58 +0200170 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200171 const VideoStreamEncoderSettings& settings,
172 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100173 : VideoStreamEncoder(Clock::GetRealTimeClock(),
174 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200175 stats_proxy,
176 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200177 std::unique_ptr<OveruseFrameDetector>(
178 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100179 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100180 task_queue_factory),
Henrik Boströmc55516d2020-05-11 16:29:22 +0200181 fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
182 fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
183 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200184 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200185 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100186 }
perkj803d97f2016-11-01 11:45:46 -0700187
kthelgason2fc52542017-03-03 00:24:41 -0800188 // This is used as a synchronisation mechanism, to make sure that the
189 // encoder queue is not blocked before we start sending it frames.
190 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100191 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200192 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800193 ASSERT_TRUE(event.Wait(5000));
194 }
195
Henrik Boström91aa7322020-04-28 12:24:33 +0200196 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200197 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200198 rtc::Event event;
199 encoder_queue()->PostTask([this, &event] {
200 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
201 event.Set();
202 });
203 ASSERT_TRUE(event.Wait(5000));
204 }
205 void TriggerCpuUnderuse() {
206 rtc::Event event;
207 encoder_queue()->PostTask([this, &event] {
208 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
209 event.Set();
210 });
211 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200212 }
kthelgason876222f2016-11-29 01:44:11 -0800213
Henrik Boström91aa7322020-04-28 12:24:33 +0200214 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200215 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200216 rtc::Event event;
217 encoder_queue()->PostTask([this, &event] {
218 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
219 event.Set();
220 });
221 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200222 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200223 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200224 rtc::Event event;
225 encoder_queue()->PostTask([this, &event] {
226 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
227 event.Set();
228 });
229 ASSERT_TRUE(event.Wait(5000));
230 }
231
232 // Fakes high QP resource usage measurements on the real
233 // QualityScalerResource. Returns whether or not QP samples would have been
234 // cleared if this had been a real signal from the QualityScaler.
235 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
236 rtc::Event event;
237 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
238 new FakeQualityScalerQpUsageHandlerCallback();
239 encoder_queue()->PostTask([this, &event, callback] {
240 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
241 event.Set();
242 });
243 EXPECT_TRUE(event.Wait(5000));
244 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
245 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200246 }
sprangfda496a2017-06-15 04:21:07 -0700247
Niels Möller7dc26b72017-12-06 10:27:48 +0100248 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200249 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
250 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700251};
252
asapersson5f7226f2016-11-25 04:37:00 -0800253class VideoStreamFactory
254 : public VideoEncoderConfig::VideoStreamFactoryInterface {
255 public:
sprangfda496a2017-06-15 04:21:07 -0700256 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
257 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800258 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700259 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800260 }
261
262 private:
263 std::vector<VideoStream> CreateEncoderStreams(
264 int width,
265 int height,
266 const VideoEncoderConfig& encoder_config) override {
267 std::vector<VideoStream> streams =
268 test::CreateVideoStreams(width, height, encoder_config);
269 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100270 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700271 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800272 }
273 return streams;
274 }
sprangfda496a2017-06-15 04:21:07 -0700275
asapersson5f7226f2016-11-25 04:37:00 -0800276 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700277 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800278};
279
Noah Richards51db4212019-06-12 06:59:12 -0700280// Simulates simulcast behavior and makes highest stream resolutions divisible
281// by 4.
282class CroppingVideoStreamFactory
283 : public VideoEncoderConfig::VideoStreamFactoryInterface {
284 public:
285 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
286 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
287 EXPECT_GT(num_temporal_layers, 0u);
288 EXPECT_GT(framerate, 0);
289 }
290
291 private:
292 std::vector<VideoStream> CreateEncoderStreams(
293 int width,
294 int height,
295 const VideoEncoderConfig& encoder_config) override {
296 std::vector<VideoStream> streams = test::CreateVideoStreams(
297 width - width % 4, height - height % 4, encoder_config);
298 for (VideoStream& stream : streams) {
299 stream.num_temporal_layers = num_temporal_layers_;
300 stream.max_framerate = framerate_;
301 }
302 return streams;
303 }
304
305 const size_t num_temporal_layers_;
306 const int framerate_;
307};
308
sprangb1ca0732017-02-01 08:38:12 -0800309class AdaptingFrameForwarder : public test::FrameForwarder {
310 public:
311 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700312 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800313
314 void set_adaptation_enabled(bool enabled) {
315 rtc::CritScope cs(&crit_);
316 adaptation_enabled_ = enabled;
317 }
318
asaperssonfab67072017-04-04 05:51:49 -0700319 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800320 rtc::CritScope cs(&crit_);
321 return adaptation_enabled_;
322 }
323
asapersson09f05612017-05-15 23:40:18 -0700324 rtc::VideoSinkWants last_wants() const {
325 rtc::CritScope cs(&crit_);
326 return last_wants_;
327 }
328
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200329 absl::optional<int> last_sent_width() const { return last_width_; }
330 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800331
sprangb1ca0732017-02-01 08:38:12 -0800332 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
333 int cropped_width = 0;
334 int cropped_height = 0;
335 int out_width = 0;
336 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700337 if (adaption_enabled()) {
338 if (adapter_.AdaptFrameResolution(
339 video_frame.width(), video_frame.height(),
340 video_frame.timestamp_us() * 1000, &cropped_width,
341 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100342 VideoFrame adapted_frame =
343 VideoFrame::Builder()
344 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
345 nullptr, out_width, out_height))
346 .set_timestamp_rtp(99)
347 .set_timestamp_ms(99)
348 .set_rotation(kVideoRotation_0)
349 .build();
sprangc5d62e22017-04-02 23:53:04 -0700350 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100351 if (video_frame.has_update_rect()) {
352 adapted_frame.set_update_rect(
353 video_frame.update_rect().ScaleWithFrame(
354 video_frame.width(), video_frame.height(), 0, 0,
355 video_frame.width(), video_frame.height(), out_width,
356 out_height));
357 }
sprangc5d62e22017-04-02 23:53:04 -0700358 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800359 last_width_.emplace(adapted_frame.width());
360 last_height_.emplace(adapted_frame.height());
361 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200362 last_width_ = absl::nullopt;
363 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700364 }
sprangb1ca0732017-02-01 08:38:12 -0800365 } else {
366 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800367 last_width_.emplace(video_frame.width());
368 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800369 }
370 }
371
372 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
373 const rtc::VideoSinkWants& wants) override {
374 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700375 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100376 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800377 test::FrameForwarder::AddOrUpdateSink(sink, wants);
378 }
sprangb1ca0732017-02-01 08:38:12 -0800379 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700380 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
381 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200382 absl::optional<int> last_width_;
383 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800384};
sprangc5d62e22017-04-02 23:53:04 -0700385
Niels Möller213618e2018-07-24 09:29:58 +0200386// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700387class MockableSendStatisticsProxy : public SendStatisticsProxy {
388 public:
389 MockableSendStatisticsProxy(Clock* clock,
390 const VideoSendStream::Config& config,
391 VideoEncoderConfig::ContentType content_type)
392 : SendStatisticsProxy(clock, config, content_type) {}
393
394 VideoSendStream::Stats GetStats() override {
395 rtc::CritScope cs(&lock_);
396 if (mock_stats_)
397 return *mock_stats_;
398 return SendStatisticsProxy::GetStats();
399 }
400
Niels Möller213618e2018-07-24 09:29:58 +0200401 int GetInputFrameRate() const override {
402 rtc::CritScope cs(&lock_);
403 if (mock_stats_)
404 return mock_stats_->input_frame_rate;
405 return SendStatisticsProxy::GetInputFrameRate();
406 }
sprangc5d62e22017-04-02 23:53:04 -0700407 void SetMockStats(const VideoSendStream::Stats& stats) {
408 rtc::CritScope cs(&lock_);
409 mock_stats_.emplace(stats);
410 }
411
412 void ResetMockStats() {
413 rtc::CritScope cs(&lock_);
414 mock_stats_.reset();
415 }
416
417 private:
418 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200419 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700420};
421
sprang4847ae62017-06-27 07:06:52 -0700422class MockBitrateObserver : public VideoBitrateAllocationObserver {
423 public:
Erik Språng566124a2018-04-23 12:32:22 +0200424 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700425};
426
philipel9b058032020-02-10 11:30:00 +0100427class MockEncoderSelector
428 : public VideoEncoderFactory::EncoderSelectorInterface {
429 public:
430 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
Mirta Dvornicic4f34d782020-02-26 13:01:19 +0100431 MOCK_METHOD1(OnAvailableBitrate,
philipel9b058032020-02-10 11:30:00 +0100432 absl::optional<SdpVideoFormat>(const DataRate& rate));
433 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
434};
435
perkj803d97f2016-11-01 11:45:46 -0700436} // namespace
437
mflodmancc3d4422017-08-03 08:27:51 -0700438class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700439 public:
440 static const int kDefaultTimeoutMs = 30 * 1000;
441
mflodmancc3d4422017-08-03 08:27:51 -0700442 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700443 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700444 codec_width_(320),
445 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200446 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200447 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700448 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200449 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700450 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700451 Clock::GetRealTimeClock(),
452 video_send_config_,
453 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700454 sink_(&fake_encoder_) {}
455
456 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700457 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700458 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200459 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800460 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200461 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200462 video_send_config_.rtp.payload_name = "FAKE";
463 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700464
Per512ecb32016-09-23 15:52:06 +0200465 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200466 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700467 video_encoder_config.video_stream_factory =
468 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100469 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700470
471 // Framerate limit is specified by the VideoStreamFactory.
472 std::vector<VideoStream> streams =
473 video_encoder_config.video_stream_factory->CreateEncoderStreams(
474 codec_width_, codec_height_, video_encoder_config);
475 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100476 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700477
Niels Möllerf1338562018-04-26 09:51:47 +0200478 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800479 }
480
Niels Möllerf1338562018-04-26 09:51:47 +0200481 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700482 if (video_stream_encoder_)
483 video_stream_encoder_->Stop();
484 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200485 stats_proxy_.get(), video_send_config_.encoder_settings,
486 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700487 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
488 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700489 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700490 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
491 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200492 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700493 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800494 }
495
496 void ResetEncoder(const std::string& payload_name,
497 size_t num_streams,
498 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700499 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700500 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200501 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800502
503 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200504 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800505 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100506 video_encoder_config.max_bitrate_bps =
507 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800508 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700509 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
510 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700511 video_encoder_config.content_type =
512 screenshare ? VideoEncoderConfig::ContentType::kScreen
513 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700514 if (payload_name == "VP9") {
515 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
516 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
517 video_encoder_config.encoder_specific_settings =
518 new rtc::RefCountedObject<
519 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
520 }
Niels Möllerf1338562018-04-26 09:51:47 +0200521 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700522 }
523
sprang57c2fff2017-01-16 06:24:02 -0800524 VideoFrame CreateFrame(int64_t ntp_time_ms,
525 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100526 VideoFrame frame =
527 VideoFrame::Builder()
528 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
529 destruction_event, codec_width_, codec_height_))
530 .set_timestamp_rtp(99)
531 .set_timestamp_ms(99)
532 .set_rotation(kVideoRotation_0)
533 .build();
sprang57c2fff2017-01-16 06:24:02 -0800534 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700535 return frame;
536 }
537
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100538 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
539 rtc::Event* destruction_event,
540 int offset_x) const {
541 VideoFrame frame =
542 VideoFrame::Builder()
543 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
544 destruction_event, codec_width_, codec_height_))
545 .set_timestamp_rtp(99)
546 .set_timestamp_ms(99)
547 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100548 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100549 .build();
550 frame.set_ntp_time_ms(ntp_time_ms);
551 return frame;
552 }
553
sprang57c2fff2017-01-16 06:24:02 -0800554 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100555 VideoFrame frame =
556 VideoFrame::Builder()
557 .set_video_frame_buffer(
558 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
559 .set_timestamp_rtp(99)
560 .set_timestamp_ms(99)
561 .set_rotation(kVideoRotation_0)
562 .build();
sprang57c2fff2017-01-16 06:24:02 -0800563 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700564 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700565 return frame;
566 }
567
Noah Richards51db4212019-06-12 06:59:12 -0700568 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
569 rtc::Event* destruction_event,
570 int width,
571 int height) const {
572 VideoFrame frame =
573 VideoFrame::Builder()
574 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
575 destruction_event, width, height))
576 .set_timestamp_rtp(99)
577 .set_timestamp_ms(99)
578 .set_rotation(kVideoRotation_0)
579 .build();
580 frame.set_ntp_time_ms(ntp_time_ms);
581 return frame;
582 }
583
584 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
585 rtc::Event* destruction_event) const {
586 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
587 codec_height_);
588 }
589
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100590 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
591 MockBitrateObserver bitrate_observer;
592 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
593
594 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
595 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200596 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100597 DataRate::BitsPerSec(kTargetBitrateBps),
598 DataRate::BitsPerSec(kTargetBitrateBps),
599 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100600
601 video_source_.IncomingCapturedFrame(
602 CreateFrame(1, codec_width_, codec_height_));
603 WaitForEncodedFrame(1);
604 }
605
asapersson02465b82017-04-10 01:12:52 -0700606 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700607 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700608 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
609 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700610 }
611
asapersson09f05612017-05-15 23:40:18 -0700612 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
613 const rtc::VideoSinkWants& wants2) {
614 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
615 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
616 }
617
Åsa Persson8c1bf952018-09-13 10:42:19 +0200618 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
619 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
620 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
621 EXPECT_FALSE(wants.target_pixel_count);
622 }
623
asapersson09f05612017-05-15 23:40:18 -0700624 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
625 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200626 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700627 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
628 EXPECT_GT(wants1.max_pixel_count, 0);
629 }
630
631 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
632 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200633 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700634 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
635 }
636
asaperssonf7e294d2017-06-13 23:25:22 -0700637 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
638 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200639 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700640 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
641 }
642
643 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
644 const rtc::VideoSinkWants& wants2) {
645 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
646 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
647 }
648
649 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
650 const rtc::VideoSinkWants& wants2) {
651 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
652 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
653 }
654
655 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
656 const rtc::VideoSinkWants& wants2) {
657 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
658 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
659 EXPECT_GT(wants1.max_pixel_count, 0);
660 }
661
662 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
663 const rtc::VideoSinkWants& wants2) {
664 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
665 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
666 }
667
asapersson09f05612017-05-15 23:40:18 -0700668 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
669 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200670 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700671 EXPECT_LT(wants.max_pixel_count, pixel_count);
672 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700673 }
674
675 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
676 EXPECT_LT(wants.max_framerate_fps, fps);
677 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
678 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700679 }
680
asaperssonf7e294d2017-06-13 23:25:22 -0700681 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
682 int expected_fps) {
683 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
684 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
685 EXPECT_FALSE(wants.target_pixel_count);
686 }
687
Jonathan Yubc771b72017-12-08 17:04:29 -0800688 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
689 int last_frame_pixels) {
690 // Balanced mode should always scale FPS to the desired range before
691 // attempting to scale resolution.
692 int fps_limit = wants.max_framerate_fps;
693 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100694 EXPECT_LE(7, fps_limit);
695 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800696 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100697 EXPECT_LE(10, fps_limit);
698 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800699 } else if (last_frame_pixels <= 640 * 480) {
700 EXPECT_LE(15, fps_limit);
701 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200702 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800703 }
704 }
705
sprang4847ae62017-06-27 07:06:52 -0700706 void WaitForEncodedFrame(int64_t expected_ntp_time) {
707 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100708 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700709 }
710
711 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
712 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100713 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700714 return ok;
715 }
716
717 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
718 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100719 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700720 }
721
722 void ExpectDroppedFrame() {
723 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100724 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700725 }
726
727 bool WaitForFrame(int64_t timeout_ms) {
728 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100729 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700730 return ok;
731 }
732
perkj26091b12016-09-01 01:17:40 -0700733 class TestEncoder : public test::FakeEncoder {
734 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100735 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700736
asaperssonfab67072017-04-04 05:51:49 -0700737 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800738 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700739 return config_;
740 }
741
742 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800743 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700744 block_next_encode_ = true;
745 }
746
Erik Språngaed30702018-11-05 12:57:17 +0100747 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800748 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100749 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100750 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100751 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100752 info.scaling_settings = VideoEncoder::ScalingSettings(
753 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100754 }
755 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100756 for (int i = 0; i < kMaxSpatialLayers; ++i) {
757 if (temporal_layers_supported_[i]) {
758 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
759 info.fps_allocation[i].resize(num_layers);
760 }
761 }
Erik Språngaed30702018-11-05 12:57:17 +0100762 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200763
764 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100765 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100766 return info;
kthelgason876222f2016-11-29 01:44:11 -0800767 }
768
Erik Språngb7cb7b52019-02-26 15:52:33 +0100769 int32_t RegisterEncodeCompleteCallback(
770 EncodedImageCallback* callback) override {
771 rtc::CritScope lock(&local_crit_sect_);
772 encoded_image_callback_ = callback;
773 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
774 }
775
perkjfa10b552016-10-02 23:45:26 -0700776 void ContinueEncode() { continue_encode_event_.Set(); }
777
778 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
779 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800780 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700781 EXPECT_EQ(timestamp_, timestamp);
782 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
783 }
784
kthelgason2fc52542017-03-03 00:24:41 -0800785 void SetQualityScaling(bool b) {
786 rtc::CritScope lock(&local_crit_sect_);
787 quality_scaling_ = b;
788 }
kthelgasonad9010c2017-02-14 00:46:51 -0800789
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100790 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
791 rtc::CritScope lock(&local_crit_sect_);
792 requested_resolution_alignment_ = requested_resolution_alignment;
793 }
794
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100795 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
796 rtc::CritScope lock(&local_crit_sect_);
797 is_hardware_accelerated_ = is_hardware_accelerated;
798 }
799
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100800 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
801 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
802 rtc::CritScope lock(&local_crit_sect_);
803 temporal_layers_supported_[spatial_idx] = supported;
804 }
805
Sergey Silkin6456e352019-07-08 17:56:40 +0200806 void SetResolutionBitrateLimits(
807 std::vector<ResolutionBitrateLimits> thresholds) {
808 rtc::CritScope cs(&local_crit_sect_);
809 resolution_bitrate_limits_ = thresholds;
810 }
811
sprangfe627f32017-03-29 08:24:59 -0700812 void ForceInitEncodeFailure(bool force_failure) {
813 rtc::CritScope lock(&local_crit_sect_);
814 force_init_encode_failed_ = force_failure;
815 }
816
Niels Möller6bb5ab92019-01-11 11:11:10 +0100817 void SimulateOvershoot(double rate_factor) {
818 rtc::CritScope lock(&local_crit_sect_);
819 rate_factor_ = rate_factor;
820 }
821
Erik Språngd7329ca2019-02-21 21:19:53 +0100822 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100823 rtc::CritScope lock(&local_crit_sect_);
824 return last_framerate_;
825 }
826
Erik Språngd7329ca2019-02-21 21:19:53 +0100827 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100828 rtc::CritScope lock(&local_crit_sect_);
829 return last_update_rect_;
830 }
831
Niels Möller87e2d782019-03-07 10:18:23 +0100832 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100833 rtc::CritScope lock(&local_crit_sect_);
834 return last_frame_types_;
835 }
836
837 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100838 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100839 keyframe ? VideoFrameType::kVideoFrameKey
840 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100841 {
842 rtc::CritScope lock(&local_crit_sect_);
843 last_frame_types_ = frame_type;
844 }
Niels Möllerb859b322019-03-07 12:40:01 +0100845 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100846 }
847
Erik Språngb7cb7b52019-02-26 15:52:33 +0100848 void InjectEncodedImage(const EncodedImage& image) {
849 rtc::CritScope lock(&local_crit_sect_);
850 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
851 }
852
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200853 void InjectEncodedImage(const EncodedImage& image,
854 const CodecSpecificInfo* codec_specific_info,
855 const RTPFragmentationHeader* fragmentation) {
856 rtc::CritScope lock(&local_crit_sect_);
857 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
858 fragmentation);
859 }
860
Erik Språngd7329ca2019-02-21 21:19:53 +0100861 void ExpectNullFrame() {
862 rtc::CritScope lock(&local_crit_sect_);
863 expect_null_frame_ = true;
864 }
865
Erik Språng5056af02019-09-02 15:53:11 +0200866 absl::optional<VideoEncoder::RateControlParameters>
867 GetAndResetLastRateControlSettings() {
868 auto settings = last_rate_control_settings_;
869 last_rate_control_settings_.reset();
870 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100871 }
872
Sergey Silkin5ee69672019-07-02 14:18:34 +0200873 int GetNumEncoderInitializations() const {
874 rtc::CritScope lock(&local_crit_sect_);
875 return num_encoder_initializations_;
876 }
877
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200878 int GetNumSetRates() const {
879 rtc::CritScope lock(&local_crit_sect_);
880 return num_set_rates_;
881 }
882
perkjfa10b552016-10-02 23:45:26 -0700883 private:
perkj26091b12016-09-01 01:17:40 -0700884 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100885 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700886 bool block_encode;
887 {
brandtre78d2662017-01-16 05:57:16 -0800888 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100889 if (expect_null_frame_) {
890 EXPECT_EQ(input_image.timestamp(), 0u);
891 EXPECT_EQ(input_image.width(), 1);
892 last_frame_types_ = *frame_types;
893 expect_null_frame_ = false;
894 } else {
895 EXPECT_GT(input_image.timestamp(), timestamp_);
896 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
897 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
898 }
perkj26091b12016-09-01 01:17:40 -0700899
900 timestamp_ = input_image.timestamp();
901 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700902 last_input_width_ = input_image.width();
903 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700904 block_encode = block_next_encode_;
905 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100906 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100907 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700908 }
Niels Möllerb859b322019-03-07 12:40:01 +0100909 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700910 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700911 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700912 return result;
913 }
914
sprangfe627f32017-03-29 08:24:59 -0700915 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200916 const Settings& settings) override {
917 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200918
sprangfe627f32017-03-29 08:24:59 -0700919 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100920 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200921
922 ++num_encoder_initializations_;
923
Erik Språng82fad3d2018-03-21 09:57:23 +0100924 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700925 // Simulate setting up temporal layers, in order to validate the life
926 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100927 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200928 frame_buffer_controller_ =
929 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700930 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100931 if (force_init_encode_failed_) {
932 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700933 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100934 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100935
Erik Språngb7cb7b52019-02-26 15:52:33 +0100936 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700937 return res;
938 }
939
Erik Språngb7cb7b52019-02-26 15:52:33 +0100940 int32_t Release() override {
941 rtc::CritScope lock(&local_crit_sect_);
942 EXPECT_NE(initialized_, EncoderState::kUninitialized);
943 initialized_ = EncoderState::kUninitialized;
944 return FakeEncoder::Release();
945 }
946
Erik Språng16cb8f52019-04-12 13:59:09 +0200947 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100948 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200949 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100950 VideoBitrateAllocation adjusted_rate_allocation;
951 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
952 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200953 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100954 adjusted_rate_allocation.SetBitrate(
955 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200956 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100957 rate_factor_));
958 }
959 }
960 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200961 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200962 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200963 RateControlParameters adjusted_paramters = parameters;
964 adjusted_paramters.bitrate = adjusted_rate_allocation;
965 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100966 }
967
brandtre78d2662017-01-16 05:57:16 -0800968 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100969 enum class EncoderState {
970 kUninitialized,
971 kInitializationFailed,
972 kInitialized
973 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
974 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700975 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700976 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700977 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
978 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
979 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
980 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
981 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100982 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100983 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100984 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700985 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100986 absl::optional<bool>
987 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
988 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700989 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100990 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
991 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200992 absl::optional<VideoEncoder::RateControlParameters>
993 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100994 VideoFrame::UpdateRect last_update_rect_
995 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100996 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100997 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100998 EncodedImageCallback* encoded_image_callback_
999 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001000 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001001 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001002 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1003 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001004 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001005 };
1006
mflodmancc3d4422017-08-03 08:27:51 -07001007 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001008 public:
1009 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001010 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001011
perkj26091b12016-09-01 01:17:40 -07001012 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001013 EXPECT_TRUE(
1014 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1015 }
1016
1017 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1018 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001019 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001020 if (!encoded_frame_event_.Wait(timeout_ms))
1021 return false;
perkj26091b12016-09-01 01:17:40 -07001022 {
1023 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001024 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001025 }
1026 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001027 return true;
perkj26091b12016-09-01 01:17:40 -07001028 }
1029
sprangb1ca0732017-02-01 08:38:12 -08001030 void WaitForEncodedFrame(uint32_t expected_width,
1031 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001032 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001033 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001034 }
1035
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001036 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001037 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001038 uint32_t width = 0;
1039 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001040 {
1041 rtc::CritScope lock(&crit_);
1042 width = last_width_;
1043 height = last_height_;
1044 }
1045 EXPECT_EQ(expected_height, height);
1046 EXPECT_EQ(expected_width, width);
1047 }
1048
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001049 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1050 int width = 0;
1051 int height = 0;
1052 {
1053 rtc::CritScope lock(&crit_);
1054 width = last_width_;
1055 height = last_height_;
1056 }
1057 EXPECT_EQ(width % resolution_alignment, 0);
1058 EXPECT_EQ(height % resolution_alignment, 0);
1059 }
1060
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001061 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1062 VideoRotation rotation;
1063 {
1064 rtc::CritScope lock(&crit_);
1065 rotation = last_rotation_;
1066 }
1067 EXPECT_EQ(expected_rotation, rotation);
1068 }
1069
kthelgason2fc52542017-03-03 00:24:41 -08001070 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001071
sprangc5d62e22017-04-02 23:53:04 -07001072 bool WaitForFrame(int64_t timeout_ms) {
1073 return encoded_frame_event_.Wait(timeout_ms);
1074 }
1075
perkj26091b12016-09-01 01:17:40 -07001076 void SetExpectNoFrames() {
1077 rtc::CritScope lock(&crit_);
1078 expect_frames_ = false;
1079 }
1080
asaperssonfab67072017-04-04 05:51:49 -07001081 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001082 rtc::CritScope lock(&crit_);
1083 return number_of_reconfigurations_;
1084 }
1085
asaperssonfab67072017-04-04 05:51:49 -07001086 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001087 rtc::CritScope lock(&crit_);
1088 return min_transmit_bitrate_bps_;
1089 }
1090
Erik Språngd7329ca2019-02-21 21:19:53 +01001091 void SetNumExpectedLayers(size_t num_layers) {
1092 rtc::CritScope lock(&crit_);
1093 num_expected_layers_ = num_layers;
1094 }
1095
Erik Språngb7cb7b52019-02-26 15:52:33 +01001096 int64_t GetLastCaptureTimeMs() const {
1097 rtc::CritScope lock(&crit_);
1098 return last_capture_time_ms_;
1099 }
1100
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001101 std::vector<uint8_t> GetLastEncodedImageData() {
1102 rtc::CritScope lock(&crit_);
1103 return std::move(last_encoded_image_data_);
1104 }
1105
1106 RTPFragmentationHeader GetLastFragmentation() {
1107 rtc::CritScope lock(&crit_);
1108 return std::move(last_fragmentation_);
1109 }
1110
perkj26091b12016-09-01 01:17:40 -07001111 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001112 Result OnEncodedImage(
1113 const EncodedImage& encoded_image,
1114 const CodecSpecificInfo* codec_specific_info,
1115 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001116 rtc::CritScope lock(&crit_);
1117 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001118 last_encoded_image_data_ = std::vector<uint8_t>(
1119 encoded_image.data(), encoded_image.data() + encoded_image.size());
1120 if (fragmentation) {
1121 last_fragmentation_.CopyFrom(*fragmentation);
1122 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001123 uint32_t timestamp = encoded_image.Timestamp();
1124 if (last_timestamp_ != timestamp) {
1125 num_received_layers_ = 1;
1126 } else {
1127 ++num_received_layers_;
1128 }
1129 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001130 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001131 last_width_ = encoded_image._encodedWidth;
1132 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001133 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001134 if (num_received_layers_ == num_expected_layers_) {
1135 encoded_frame_event_.Set();
1136 }
sprangb1ca0732017-02-01 08:38:12 -08001137 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001138 }
1139
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001140 void OnEncoderConfigurationChanged(
1141 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001142 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001143 VideoEncoderConfig::ContentType content_type,
1144 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001145 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001146 ++number_of_reconfigurations_;
1147 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1148 }
1149
perkj26091b12016-09-01 01:17:40 -07001150 rtc::CriticalSection crit_;
1151 TestEncoder* test_encoder_;
1152 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001153 std::vector<uint8_t> last_encoded_image_data_;
1154 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001155 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001156 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001157 uint32_t last_height_ = 0;
1158 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001159 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001160 size_t num_expected_layers_ = 1;
1161 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001162 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001163 int number_of_reconfigurations_ = 0;
1164 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001165 };
1166
Sergey Silkin5ee69672019-07-02 14:18:34 +02001167 class VideoBitrateAllocatorProxyFactory
1168 : public VideoBitrateAllocatorFactory {
1169 public:
1170 VideoBitrateAllocatorProxyFactory()
1171 : bitrate_allocator_factory_(
1172 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1173
1174 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1175 const VideoCodec& codec) override {
1176 rtc::CritScope lock(&crit_);
1177 codec_config_ = codec;
1178 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1179 }
1180
1181 VideoCodec codec_config() const {
1182 rtc::CritScope lock(&crit_);
1183 return codec_config_;
1184 }
1185
1186 private:
1187 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1188
1189 rtc::CriticalSection crit_;
1190 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1191 };
1192
perkj26091b12016-09-01 01:17:40 -07001193 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001194 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001195 int codec_width_;
1196 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001197 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001198 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001199 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001200 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001201 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001202 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001203 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001204 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001205 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001206 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001207};
1208
mflodmancc3d4422017-08-03 08:27:51 -07001209TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001210 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001211 DataRate::BitsPerSec(kTargetBitrateBps),
1212 DataRate::BitsPerSec(kTargetBitrateBps),
1213 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001214 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001215 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001216 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001217 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001218 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001219}
1220
mflodmancc3d4422017-08-03 08:27:51 -07001221TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001222 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001223 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001224 // The encoder will cache up to one frame for a short duration. Adding two
1225 // frames means that the first frame will be dropped and the second frame will
1226 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001227 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001228 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001229 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001230
Erik Språng4c6ca302019-04-08 15:14:01 +02001231 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001232 DataRate::BitsPerSec(kTargetBitrateBps),
1233 DataRate::BitsPerSec(kTargetBitrateBps),
1234 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001235
Sebastian Janssona3177052018-04-10 13:05:49 +02001236 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001237 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001238 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1239
1240 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001241 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001242}
1243
mflodmancc3d4422017-08-03 08:27:51 -07001244TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001245 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001246 DataRate::BitsPerSec(kTargetBitrateBps),
1247 DataRate::BitsPerSec(kTargetBitrateBps),
1248 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001249 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001250 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001251
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001252 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
1253 DataRate::BitsPerSec(0),
1254 DataRate::BitsPerSec(0), 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001255 // The encoder will cache up to one frame for a short duration. Adding two
1256 // frames means that the first frame will be dropped and the second frame will
1257 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001258 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001259 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001260
Erik Språng4c6ca302019-04-08 15:14:01 +02001261 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001262 DataRate::BitsPerSec(kTargetBitrateBps),
1263 DataRate::BitsPerSec(kTargetBitrateBps),
1264 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001265 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001266 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1267 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001268 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001269}
1270
mflodmancc3d4422017-08-03 08:27:51 -07001271TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001272 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001273 DataRate::BitsPerSec(kTargetBitrateBps),
1274 DataRate::BitsPerSec(kTargetBitrateBps),
1275 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001276 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001277 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001278
1279 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001280 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001281
perkja49cbd32016-09-16 07:53:41 -07001282 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001283 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001284 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001285}
1286
mflodmancc3d4422017-08-03 08:27:51 -07001287TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001288 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001289 DataRate::BitsPerSec(kTargetBitrateBps),
1290 DataRate::BitsPerSec(kTargetBitrateBps),
1291 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001292
perkja49cbd32016-09-16 07:53:41 -07001293 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001294 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001295
mflodmancc3d4422017-08-03 08:27:51 -07001296 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001297 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001298 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001299 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1300 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001301}
1302
mflodmancc3d4422017-08-03 08:27:51 -07001303TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001304 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001305 DataRate::BitsPerSec(kTargetBitrateBps),
1306 DataRate::BitsPerSec(kTargetBitrateBps),
1307 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001308
1309 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001310 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001311 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001312 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1313 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001314 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1315 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001316 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001317 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001318
mflodmancc3d4422017-08-03 08:27:51 -07001319 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001320}
1321
Noah Richards51db4212019-06-12 06:59:12 -07001322TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1323 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001324 DataRate::BitsPerSec(kTargetBitrateBps),
1325 DataRate::BitsPerSec(kTargetBitrateBps),
1326 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001327
1328 rtc::Event frame_destroyed_event;
1329 video_source_.IncomingCapturedFrame(
1330 CreateFakeNativeFrame(1, &frame_destroyed_event));
1331 ExpectDroppedFrame();
1332 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1333 video_stream_encoder_->Stop();
1334}
1335
1336TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1337 // Use the cropping factory.
1338 video_encoder_config_.video_stream_factory =
1339 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1340 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1341 kMaxPayloadLength);
1342 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1343
1344 // Capture a frame at codec_width_/codec_height_.
1345 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001346 DataRate::BitsPerSec(kTargetBitrateBps),
1347 DataRate::BitsPerSec(kTargetBitrateBps),
1348 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001349 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1350 WaitForEncodedFrame(1);
1351 // The encoder will have been configured once.
1352 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1353 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1354 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1355
1356 // Now send in a fake frame that needs to be cropped as the width/height
1357 // aren't divisible by 4 (see CreateEncoderStreams above).
1358 rtc::Event frame_destroyed_event;
1359 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1360 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1361 ExpectDroppedFrame();
1362 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1363 video_stream_encoder_->Stop();
1364}
1365
Ying Wang9b881ab2020-02-07 14:29:32 +01001366TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
1367 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001368 DataRate::BitsPerSec(kTargetBitrateBps),
1369 DataRate::BitsPerSec(kTargetBitrateBps),
1370 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001371 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1372 WaitForEncodedFrame(1);
1373
1374 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001375 DataRate::BitsPerSec(kTargetBitrateBps),
1376 DataRate::BitsPerSec(kTargetBitrateBps),
1377 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001378 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1379 // frames. Adding two frames means that the first frame will be dropped and
1380 // the second frame will be sent to the encoder.
1381 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1382 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1383 WaitForEncodedFrame(3);
1384 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1385 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1386 WaitForEncodedFrame(5);
1387 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1388 video_stream_encoder_->Stop();
1389}
1390
mflodmancc3d4422017-08-03 08:27:51 -07001391TEST_F(VideoStreamEncoderTest,
1392 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001393 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001394 DataRate::BitsPerSec(kTargetBitrateBps),
1395 DataRate::BitsPerSec(kTargetBitrateBps),
1396 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001397 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001398
1399 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001400 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001401 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001402 // The encoder will have been configured once when the first frame is
1403 // received.
1404 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001405
1406 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001407 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001408 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001409 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001410 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001411
1412 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001413 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001414 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001415 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001416 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001417
mflodmancc3d4422017-08-03 08:27:51 -07001418 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001419}
1420
mflodmancc3d4422017-08-03 08:27:51 -07001421TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001422 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001423 DataRate::BitsPerSec(kTargetBitrateBps),
1424 DataRate::BitsPerSec(kTargetBitrateBps),
1425 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001426
1427 // Capture a frame and wait for it to synchronize with the encoder thread.
1428 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001429 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001430 // The encoder will have been configured once.
1431 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001432 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1433 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1434
1435 codec_width_ *= 2;
1436 codec_height_ *= 2;
1437 // Capture a frame with a higher resolution and wait for it to synchronize
1438 // with the encoder thread.
1439 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001440 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001441 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1442 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001443 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001444
mflodmancc3d4422017-08-03 08:27:51 -07001445 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001446}
1447
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001448TEST_F(VideoStreamEncoderTest,
1449 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1450 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001451 DataRate::BitsPerSec(kTargetBitrateBps),
1452 DataRate::BitsPerSec(kTargetBitrateBps),
1453 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001454
1455 // Capture a frame and wait for it to synchronize with the encoder thread.
1456 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1457 WaitForEncodedFrame(1);
1458
1459 VideoEncoderConfig video_encoder_config;
1460 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1461 // Changing the max payload data length recreates encoder.
1462 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1463 kMaxPayloadLength / 2);
1464
1465 // Capture a frame and wait for it to synchronize with the encoder thread.
1466 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1467 WaitForEncodedFrame(2);
1468 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1469
1470 video_stream_encoder_->Stop();
1471}
1472
Sergey Silkin5ee69672019-07-02 14:18:34 +02001473TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1474 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001475 DataRate::BitsPerSec(kTargetBitrateBps),
1476 DataRate::BitsPerSec(kTargetBitrateBps),
1477 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001478
1479 VideoEncoderConfig video_encoder_config;
1480 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1481 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1482 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1483 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1484 kMaxPayloadLength);
1485
1486 // Capture a frame and wait for it to synchronize with the encoder thread.
1487 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1488 WaitForEncodedFrame(1);
1489 // The encoder will have been configured once when the first frame is
1490 // received.
1491 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1492 EXPECT_EQ(kTargetBitrateBps,
1493 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1494 EXPECT_EQ(kStartBitrateBps,
1495 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1496
Sergey Silkin6456e352019-07-08 17:56:40 +02001497 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1498 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001499 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1500 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1501 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1502 kMaxPayloadLength);
1503
1504 // Capture a frame and wait for it to synchronize with the encoder thread.
1505 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1506 WaitForEncodedFrame(2);
1507 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1508 // Bitrate limits have changed - rate allocator should be reconfigured,
1509 // encoder should not be reconfigured.
1510 EXPECT_EQ(kTargetBitrateBps * 2,
1511 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1512 EXPECT_EQ(kStartBitrateBps * 2,
1513 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1514 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1515
1516 video_stream_encoder_->Stop();
1517}
1518
Sergey Silkin6456e352019-07-08 17:56:40 +02001519TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001520 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001521 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001522 DataRate::BitsPerSec(kTargetBitrateBps),
1523 DataRate::BitsPerSec(kTargetBitrateBps),
1524 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001525
Sergey Silkincd02eba2020-01-20 14:48:40 +01001526 const uint32_t kMinEncBitrateKbps = 100;
1527 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001528 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001529 /*frame_size_pixels=*/codec_width_ * codec_height_,
1530 /*min_start_bitrate_bps=*/0,
1531 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1532 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001533 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1534
Sergey Silkincd02eba2020-01-20 14:48:40 +01001535 VideoEncoderConfig video_encoder_config;
1536 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1537 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1538 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1539 (kMinEncBitrateKbps + 1) * 1000;
1540 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1541 kMaxPayloadLength);
1542
1543 // When both encoder and app provide bitrate limits, the intersection of
1544 // provided sets should be used.
1545 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1546 WaitForEncodedFrame(1);
1547 EXPECT_EQ(kMaxEncBitrateKbps,
1548 bitrate_allocator_factory_.codec_config().maxBitrate);
1549 EXPECT_EQ(kMinEncBitrateKbps + 1,
1550 bitrate_allocator_factory_.codec_config().minBitrate);
1551
1552 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1553 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1554 (kMinEncBitrateKbps - 1) * 1000;
1555 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1556 kMaxPayloadLength);
1557 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001558 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001559 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001560 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001561 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001562 bitrate_allocator_factory_.codec_config().minBitrate);
1563
Sergey Silkincd02eba2020-01-20 14:48:40 +01001564 video_stream_encoder_->Stop();
1565}
1566
1567TEST_F(VideoStreamEncoderTest,
1568 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
1569 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001570 DataRate::BitsPerSec(kTargetBitrateBps),
1571 DataRate::BitsPerSec(kTargetBitrateBps),
1572 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001573
1574 const uint32_t kMinAppBitrateKbps = 100;
1575 const uint32_t kMaxAppBitrateKbps = 200;
1576 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1577 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1578 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1579 /*frame_size_pixels=*/codec_width_ * codec_height_,
1580 /*min_start_bitrate_bps=*/0,
1581 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1582 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1583 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1584
1585 VideoEncoderConfig video_encoder_config;
1586 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1587 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1588 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1589 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001590 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1591 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001592
Sergey Silkincd02eba2020-01-20 14:48:40 +01001593 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1594 WaitForEncodedFrame(1);
1595 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001596 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001597 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001598 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001599
1600 video_stream_encoder_->Stop();
1601}
1602
1603TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001604 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001605 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001606 DataRate::BitsPerSec(kTargetBitrateBps),
1607 DataRate::BitsPerSec(kTargetBitrateBps),
1608 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001609
1610 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001611 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001612 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001613 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001614 fake_encoder_.SetResolutionBitrateLimits(
1615 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1616
1617 VideoEncoderConfig video_encoder_config;
1618 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1619 video_encoder_config.max_bitrate_bps = 0;
1620 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1621 kMaxPayloadLength);
1622
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001623 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001624 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1625 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001626 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1627 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001628 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1629 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1630
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001631 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001632 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1633 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001634 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1635 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001636 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1637 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1638
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001639 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001640 // encoder for 360p should be used.
1641 video_source_.IncomingCapturedFrame(
1642 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1643 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001644 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1645 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001646 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1647 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1648
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001649 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001650 // ignored.
1651 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1652 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001653 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1654 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001655 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1656 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001657 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1658 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001659 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1660 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1661
1662 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1663 // for 270p should be used.
1664 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1665 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001666 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1667 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001668 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1669 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1670
1671 video_stream_encoder_->Stop();
1672}
1673
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001674TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1675 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001676 DataRate::BitsPerSec(kTargetBitrateBps),
1677 DataRate::BitsPerSec(kTargetBitrateBps),
1678 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001679
1680 VideoEncoderConfig video_encoder_config;
1681 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1682 video_encoder_config.max_bitrate_bps = 0;
1683 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1684 kMaxPayloadLength);
1685
1686 // Encode 720p frame to get the default encoder target bitrate.
1687 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1688 WaitForEncodedFrame(1);
1689 const uint32_t kDefaultTargetBitrateFor720pKbps =
1690 bitrate_allocator_factory_.codec_config()
1691 .simulcastStream[0]
1692 .targetBitrate;
1693
1694 // Set the max recommended encoder bitrate to something lower than the default
1695 // target bitrate.
1696 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1697 1280 * 720, 10 * 1000, 10 * 1000,
1698 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1699 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1700
1701 // Change resolution to trigger encoder reinitialization.
1702 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1703 WaitForEncodedFrame(2);
1704 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1705 WaitForEncodedFrame(3);
1706
1707 // Ensure the target bitrate is capped by the max bitrate.
1708 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1709 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1710 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1711 .simulcastStream[0]
1712 .targetBitrate *
1713 1000,
1714 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1715
1716 video_stream_encoder_->Stop();
1717}
1718
mflodmancc3d4422017-08-03 08:27:51 -07001719TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001720 EXPECT_TRUE(video_source_.has_sinks());
1721 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001722 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001723 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001724 EXPECT_FALSE(video_source_.has_sinks());
1725 EXPECT_TRUE(new_video_source.has_sinks());
1726
mflodmancc3d4422017-08-03 08:27:51 -07001727 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001728}
1729
mflodmancc3d4422017-08-03 08:27:51 -07001730TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001731 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001732 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001733 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001734 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001735}
1736
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001737TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1738 constexpr int kRequestedResolutionAlignment = 7;
1739 video_source_.set_adaptation_enabled(true);
1740 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
1741 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001742 DataRate::BitsPerSec(kTargetBitrateBps),
1743 DataRate::BitsPerSec(kTargetBitrateBps),
1744 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001745
1746 // On the 1st frame, we should have initialized the encoder and
1747 // asked for its resolution requirements.
1748 video_source_.IncomingCapturedFrame(
1749 CreateFrame(1, codec_width_, codec_height_));
1750 WaitForEncodedFrame(1);
1751 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1752 kRequestedResolutionAlignment);
1753
1754 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1755 // (It's up the to the encoder to potentially drop the previous frame,
1756 // to avoid coding back-to-back keyframes.)
1757 video_source_.IncomingCapturedFrame(
1758 CreateFrame(2, codec_width_, codec_height_));
1759 WaitForEncodedFrame(2);
1760 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1761
1762 video_stream_encoder_->Stop();
1763}
1764
Jonathan Yubc771b72017-12-08 17:04:29 -08001765TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1766 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001767 const int kWidth = 1280;
1768 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001769
1770 // We rely on the automatic resolution adaptation, but we handle framerate
1771 // adaptation manually by mocking the stats proxy.
1772 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001773
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001774 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001775 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001776 DataRate::BitsPerSec(kTargetBitrateBps),
1777 DataRate::BitsPerSec(kTargetBitrateBps),
1778 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001779 video_stream_encoder_->SetSource(&video_source_,
1780 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001781 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001782 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001784 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1785
Jonathan Yubc771b72017-12-08 17:04:29 -08001786 // Adapt down as far as possible.
1787 rtc::VideoSinkWants last_wants;
1788 int64_t t = 1;
1789 int loop_count = 0;
1790 do {
1791 ++loop_count;
1792 last_wants = video_source_.sink_wants();
1793
1794 // Simulate the framerate we've been asked to adapt to.
1795 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1796 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1797 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1798 mock_stats.input_frame_rate = fps;
1799 stats_proxy_->SetMockStats(mock_stats);
1800
1801 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1802 sink_.WaitForEncodedFrame(t);
1803 t += frame_interval_ms;
1804
mflodmancc3d4422017-08-03 08:27:51 -07001805 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001806 VerifyBalancedModeFpsRange(
1807 video_source_.sink_wants(),
1808 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1809 } while (video_source_.sink_wants().max_pixel_count <
1810 last_wants.max_pixel_count ||
1811 video_source_.sink_wants().max_framerate_fps <
1812 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001813
Jonathan Yubc771b72017-12-08 17:04:29 -08001814 // Verify that we've adapted all the way down.
1815 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001816 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001817 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1818 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001819 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001820 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1821 *video_source_.last_sent_height());
1822 EXPECT_EQ(kMinBalancedFramerateFps,
1823 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001824
Jonathan Yubc771b72017-12-08 17:04:29 -08001825 // Adapt back up the same number of times we adapted down.
1826 for (int i = 0; i < loop_count - 1; ++i) {
1827 last_wants = video_source_.sink_wants();
1828
1829 // Simulate the framerate we've been asked to adapt to.
1830 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1831 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1832 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1833 mock_stats.input_frame_rate = fps;
1834 stats_proxy_->SetMockStats(mock_stats);
1835
1836 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1837 sink_.WaitForEncodedFrame(t);
1838 t += frame_interval_ms;
1839
Henrik Boström91aa7322020-04-28 12:24:33 +02001840 video_stream_encoder_->TriggerCpuUnderuse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001841 VerifyBalancedModeFpsRange(
1842 video_source_.sink_wants(),
1843 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1844 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1845 last_wants.max_pixel_count ||
1846 video_source_.sink_wants().max_framerate_fps >
1847 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001848 }
1849
Åsa Persson8c1bf952018-09-13 10:42:19 +02001850 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001851 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001852 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1854 EXPECT_EQ((loop_count - 1) * 2,
1855 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001856
mflodmancc3d4422017-08-03 08:27:51 -07001857 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001858}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001859
mflodmancc3d4422017-08-03 08:27:51 -07001860TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001861 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001862 DataRate::BitsPerSec(kTargetBitrateBps),
1863 DataRate::BitsPerSec(kTargetBitrateBps),
1864 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001865 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001866
sprangc5d62e22017-04-02 23:53:04 -07001867 const int kFrameWidth = 1280;
1868 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001869
Åsa Persson8c1bf952018-09-13 10:42:19 +02001870 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001871
kthelgason5e13d412016-12-01 03:59:51 -08001872 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001873 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001874 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001875 frame_timestamp += kFrameIntervalMs;
1876
perkj803d97f2016-11-01 11:45:46 -07001877 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001878 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001879 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001880 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001881 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001882 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001883
asapersson0944a802017-04-07 00:57:58 -07001884 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001885 // wanted resolution.
1886 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1887 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1888 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001889 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001890
1891 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001892 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001893 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001894 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001895 // Give the encoder queue time to process the change in degradation preference
1896 // by waiting for an encoded frame.
1897 new_video_source.IncomingCapturedFrame(
1898 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1899 sink_.WaitForEncodedFrame(frame_timestamp);
1900 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001901 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001902 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001903
sprangc5d62e22017-04-02 23:53:04 -07001904 // Force an input frame rate to be available, or the adaptation call won't
1905 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001906 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001907 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001908 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001909 stats_proxy_->SetMockStats(stats);
1910
mflodmancc3d4422017-08-03 08:27:51 -07001911 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001912 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001913 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001914 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001915 frame_timestamp += kFrameIntervalMs;
1916
1917 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001918 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001919 EXPECT_EQ(std::numeric_limits<int>::max(),
1920 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001921 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001922
asapersson02465b82017-04-10 01:12:52 -07001923 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001924 video_stream_encoder_->SetSource(&new_video_source,
1925 webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01001926 // Give the encoder queue time to process the change in degradation preference
1927 // by waiting for an encoded frame.
1928 new_video_source.IncomingCapturedFrame(
1929 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1930 sink_.WaitForEncodedFrame(frame_timestamp);
1931 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001932 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001933
mflodmancc3d4422017-08-03 08:27:51 -07001934 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001935 new_video_source.IncomingCapturedFrame(
1936 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001937 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001938 frame_timestamp += kFrameIntervalMs;
1939
1940 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001941 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001942
1943 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001944 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001945 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01001946 // Give the encoder queue time to process the change in degradation preference
1947 // by waiting for an encoded frame.
1948 new_video_source.IncomingCapturedFrame(
1949 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1950 sink_.WaitForEncodedFrame(frame_timestamp);
1951 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001952 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1953 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001954 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001955 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001956
1957 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001958 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001959 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001960 // Give the encoder queue time to process the change in degradation preference
1961 // by waiting for an encoded frame.
1962 new_video_source.IncomingCapturedFrame(
1963 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1964 sink_.WaitForEncodedFrame(frame_timestamp);
1965 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001966 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1967 EXPECT_EQ(std::numeric_limits<int>::max(),
1968 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001969 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001970
mflodmancc3d4422017-08-03 08:27:51 -07001971 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001972}
1973
mflodmancc3d4422017-08-03 08:27:51 -07001974TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001975 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001976 DataRate::BitsPerSec(kTargetBitrateBps),
1977 DataRate::BitsPerSec(kTargetBitrateBps),
1978 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001979
asaperssonfab67072017-04-04 05:51:49 -07001980 const int kWidth = 1280;
1981 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001982 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001983 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001984 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1985 EXPECT_FALSE(stats.bw_limited_resolution);
1986 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1987
1988 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001989 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001990 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001991 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001992
1993 stats = stats_proxy_->GetStats();
1994 EXPECT_TRUE(stats.bw_limited_resolution);
1995 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1996
1997 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001998 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001999 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002000 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002001
2002 stats = stats_proxy_->GetStats();
2003 EXPECT_FALSE(stats.bw_limited_resolution);
2004 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2005 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2006
mflodmancc3d4422017-08-03 08:27:51 -07002007 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002008}
2009
mflodmancc3d4422017-08-03 08:27:51 -07002010TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002011 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002012 DataRate::BitsPerSec(kTargetBitrateBps),
2013 DataRate::BitsPerSec(kTargetBitrateBps),
2014 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002015
2016 const int kWidth = 1280;
2017 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002018 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002019 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002020 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2021 EXPECT_FALSE(stats.cpu_limited_resolution);
2022 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2023
2024 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002025 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002026 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002027 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002028
2029 stats = stats_proxy_->GetStats();
2030 EXPECT_TRUE(stats.cpu_limited_resolution);
2031 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2032
2033 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002034 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002035 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002036 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002037
2038 stats = stats_proxy_->GetStats();
2039 EXPECT_FALSE(stats.cpu_limited_resolution);
2040 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002041 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002042
mflodmancc3d4422017-08-03 08:27:51 -07002043 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002044}
2045
mflodmancc3d4422017-08-03 08:27:51 -07002046TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002047 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002048 DataRate::BitsPerSec(kTargetBitrateBps),
2049 DataRate::BitsPerSec(kTargetBitrateBps),
2050 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002051
asaperssonfab67072017-04-04 05:51:49 -07002052 const int kWidth = 1280;
2053 const int kHeight = 720;
2054 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002055 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002056 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002057 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002058 EXPECT_FALSE(stats.cpu_limited_resolution);
2059 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2060
asaperssonfab67072017-04-04 05:51:49 -07002061 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002062 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002063 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002064 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002065 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002066 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002067 EXPECT_TRUE(stats.cpu_limited_resolution);
2068 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2069
2070 // Set new source with adaptation still enabled.
2071 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002072 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002073 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002074
asaperssonfab67072017-04-04 05:51:49 -07002075 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002076 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002077 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002078 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002079 EXPECT_TRUE(stats.cpu_limited_resolution);
2080 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2081
2082 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002083 video_stream_encoder_->SetSource(&new_video_source,
2084 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002085
asaperssonfab67072017-04-04 05:51:49 -07002086 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002087 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002088 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002089 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002090 EXPECT_FALSE(stats.cpu_limited_resolution);
2091 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2092
2093 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002094 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002095 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002096
asaperssonfab67072017-04-04 05:51:49 -07002097 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002098 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002099 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002100 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002101 EXPECT_TRUE(stats.cpu_limited_resolution);
2102 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2103
asaperssonfab67072017-04-04 05:51:49 -07002104 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002105 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002106 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002107 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002108 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002109 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002110 EXPECT_FALSE(stats.cpu_limited_resolution);
2111 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002112 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002113
mflodmancc3d4422017-08-03 08:27:51 -07002114 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002115}
2116
mflodmancc3d4422017-08-03 08:27:51 -07002117TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002118 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002119 DataRate::BitsPerSec(kTargetBitrateBps),
2120 DataRate::BitsPerSec(kTargetBitrateBps),
2121 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002122
asaperssonfab67072017-04-04 05:51:49 -07002123 const int kWidth = 1280;
2124 const int kHeight = 720;
2125 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002126 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002127 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002128 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002129 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002130 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002131
2132 // Set new source with adaptation still enabled.
2133 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002134 video_stream_encoder_->SetSource(&new_video_source,
2135 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002136
asaperssonfab67072017-04-04 05:51:49 -07002137 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002138 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002139 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002140 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002141 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002142 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002143
asaperssonfab67072017-04-04 05:51:49 -07002144 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002145 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002146 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002147 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002148 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002149 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002150 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002151 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002152
asaperssonfab67072017-04-04 05:51:49 -07002153 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002154 video_stream_encoder_->SetSource(&new_video_source,
2155 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002156
asaperssonfab67072017-04-04 05:51:49 -07002157 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002158 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002159 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002160 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002161 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002162 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002163
asapersson02465b82017-04-10 01:12:52 -07002164 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002165 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002166 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002167
asaperssonfab67072017-04-04 05:51:49 -07002168 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002169 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002170 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002171 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002172 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002173 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2174 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002175
mflodmancc3d4422017-08-03 08:27:51 -07002176 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002177}
2178
mflodmancc3d4422017-08-03 08:27:51 -07002179TEST_F(VideoStreamEncoderTest,
2180 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002181 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002182 DataRate::BitsPerSec(kTargetBitrateBps),
2183 DataRate::BitsPerSec(kTargetBitrateBps),
2184 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002185
2186 const int kWidth = 1280;
2187 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002188 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002189 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002190 video_source_.IncomingCapturedFrame(
2191 CreateFrame(timestamp_ms, kWidth, kHeight));
2192 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002193 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2195 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2196
2197 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002198 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002199 timestamp_ms += kFrameIntervalMs;
2200 video_source_.IncomingCapturedFrame(
2201 CreateFrame(timestamp_ms, kWidth, kHeight));
2202 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002203 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2204 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2205 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2206
2207 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002208 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002209 timestamp_ms += kFrameIntervalMs;
2210 video_source_.IncomingCapturedFrame(
2211 CreateFrame(timestamp_ms, kWidth, kHeight));
2212 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002213 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2214 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2215 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2216
Niels Möller4db138e2018-04-19 09:04:13 +02002217 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002218 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002219
2220 VideoEncoderConfig video_encoder_config;
2221 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2222 // Make format different, to force recreation of encoder.
2223 video_encoder_config.video_format.parameters["foo"] = "foo";
2224 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002225 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002226 timestamp_ms += kFrameIntervalMs;
2227 video_source_.IncomingCapturedFrame(
2228 CreateFrame(timestamp_ms, kWidth, kHeight));
2229 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002230 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2231 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2232 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2233
mflodmancc3d4422017-08-03 08:27:51 -07002234 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002235}
2236
mflodmancc3d4422017-08-03 08:27:51 -07002237TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002238 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
2239 video_stream_encoder_->OnBitrateUpdated(
2240 DataRate::BitsPerSec(kTargetBitrateBps),
2241 DataRate::BitsPerSec(kTargetBitrateBps),
2242 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2243
2244 const int kWidth = 1280;
2245 const int kHeight = 720;
2246 int sequence = 1;
2247
2248 // Enable BALANCED preference, no initial limitation.
2249 test::FrameForwarder source;
2250 video_stream_encoder_->SetSource(&source,
2251 webrtc::DegradationPreference::BALANCED);
2252 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2253 WaitForEncodedFrame(sequence++);
2254 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2255 EXPECT_FALSE(stats.cpu_limited_resolution);
2256 EXPECT_FALSE(stats.cpu_limited_framerate);
2257 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2258
2259 // Trigger CPU overuse, should now adapt down.
2260 video_stream_encoder_->TriggerCpuOveruse();
2261 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2262 WaitForEncodedFrame(sequence++);
2263 stats = stats_proxy_->GetStats();
2264 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2265
2266 // Set new degradation preference should clear restrictions since we changed
2267 // from BALANCED.
2268 video_stream_encoder_->SetSource(
2269 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2270 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2271 WaitForEncodedFrame(sequence++);
2272 stats = stats_proxy_->GetStats();
2273 EXPECT_FALSE(stats.cpu_limited_resolution);
2274 EXPECT_FALSE(stats.cpu_limited_framerate);
2275 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2276
2277 // Force an input frame rate to be available, or the adaptation call won't
2278 // know what framerate to adapt from.
2279 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2280 mock_stats.input_frame_rate = 30;
2281 stats_proxy_->SetMockStats(mock_stats);
2282 video_stream_encoder_->TriggerCpuOveruse();
2283 stats_proxy_->ResetMockStats();
2284 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2285 WaitForEncodedFrame(sequence++);
2286
2287 // We have now adapted once.
2288 stats = stats_proxy_->GetStats();
2289 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2290
2291 // Back to BALANCED, should clear the restrictions again.
2292 video_stream_encoder_->SetSource(&source,
2293 webrtc::DegradationPreference::BALANCED);
2294 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2295 WaitForEncodedFrame(sequence++);
2296 stats = stats_proxy_->GetStats();
2297 EXPECT_FALSE(stats.cpu_limited_resolution);
2298 EXPECT_FALSE(stats.cpu_limited_framerate);
2299 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2300
2301 video_stream_encoder_->Stop();
2302}
2303
2304TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002305 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002306 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002307 DataRate::BitsPerSec(kTargetBitrateBps),
2308 DataRate::BitsPerSec(kTargetBitrateBps),
2309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002310
asapersson0944a802017-04-07 00:57:58 -07002311 const int kWidth = 1280;
2312 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002313 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002314
asaperssonfab67072017-04-04 05:51:49 -07002315 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002316 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002317 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002318 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002319 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002320 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2321
asapersson02465b82017-04-10 01:12:52 -07002322 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002323 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002324 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002325 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002326 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002327 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002328 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002329 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2330
2331 // Set new source with adaptation still enabled.
2332 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002334 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002335
2336 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002337 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002338 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002339 stats = stats_proxy_->GetStats();
2340 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002341 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002342 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2343
sprangc5d62e22017-04-02 23:53:04 -07002344 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002345 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002346 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002347 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002348 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002349 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002350 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002351 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002352 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002353 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002354 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2355
sprangc5d62e22017-04-02 23:53:04 -07002356 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002357 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002358 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2359 mock_stats.input_frame_rate = 30;
2360 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002361 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002362 stats_proxy_->ResetMockStats();
2363
2364 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002365 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002366 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002367
2368 // Framerate now adapted.
2369 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002370 EXPECT_FALSE(stats.cpu_limited_resolution);
2371 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002372 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2373
2374 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002375 video_stream_encoder_->SetSource(&new_video_source,
2376 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002377 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002378 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002379 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002380
2381 stats = stats_proxy_->GetStats();
2382 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002383 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002384 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2385
2386 // Try to trigger overuse. Should not succeed.
2387 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002388 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002389 stats_proxy_->ResetMockStats();
2390
2391 stats = stats_proxy_->GetStats();
2392 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002393 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002394 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2395
2396 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002397 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002398 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002399 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002400 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002401 stats = stats_proxy_->GetStats();
2402 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002403 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002404 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002405
2406 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002407 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002408 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002409 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002410 stats = stats_proxy_->GetStats();
2411 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002412 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002413 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2414
2415 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002416 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002417 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002418 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002419 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002420 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002421 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002422 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002423 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002424 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002425 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2426
2427 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002428 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002429 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002430 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002431 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002432 stats = stats_proxy_->GetStats();
2433 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002434 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002435 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002436 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002437
mflodmancc3d4422017-08-03 08:27:51 -07002438 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002439}
2440
mflodmancc3d4422017-08-03 08:27:51 -07002441TEST_F(VideoStreamEncoderTest,
2442 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002443 const int kWidth = 1280;
2444 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002445 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002446 DataRate::BitsPerSec(kTargetBitrateBps),
2447 DataRate::BitsPerSec(kTargetBitrateBps),
2448 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002449
asaperssonfab67072017-04-04 05:51:49 -07002450 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002451 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002452
asaperssonfab67072017-04-04 05:51:49 -07002453 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002454 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002455
asaperssonfab67072017-04-04 05:51:49 -07002456 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002457 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002458
asaperssonfab67072017-04-04 05:51:49 -07002459 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002460 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002461
kthelgason876222f2016-11-29 01:44:11 -08002462 // Expect a scale down.
2463 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002464 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002465
asapersson02465b82017-04-10 01:12:52 -07002466 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002467 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002468 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002469 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002470
asaperssonfab67072017-04-04 05:51:49 -07002471 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002472 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002473 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002474 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002475
asaperssonfab67072017-04-04 05:51:49 -07002476 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002477 EXPECT_EQ(std::numeric_limits<int>::max(),
2478 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002479
asaperssonfab67072017-04-04 05:51:49 -07002480 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002481 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002482 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002483 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002484
asapersson02465b82017-04-10 01:12:52 -07002485 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002486 EXPECT_EQ(std::numeric_limits<int>::max(),
2487 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002488
mflodmancc3d4422017-08-03 08:27:51 -07002489 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002490}
2491
mflodmancc3d4422017-08-03 08:27:51 -07002492TEST_F(VideoStreamEncoderTest,
2493 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002494 const int kWidth = 1280;
2495 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002496 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002497 DataRate::BitsPerSec(kTargetBitrateBps),
2498 DataRate::BitsPerSec(kTargetBitrateBps),
2499 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002500
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002501 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002502 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002503 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002504 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002505
2506 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002507 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002508 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002509 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2510 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2511
2512 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002513 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002514 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002515 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2516 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2517 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2518
2519 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002520 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002521 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2522 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2523 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2524
mflodmancc3d4422017-08-03 08:27:51 -07002525 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002526}
2527
mflodmancc3d4422017-08-03 08:27:51 -07002528TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002529 const int kWidth = 1280;
2530 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002531 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002532 DataRate::BitsPerSec(kTargetBitrateBps),
2533 DataRate::BitsPerSec(kTargetBitrateBps),
2534 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002535
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002536 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002537 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002538 video_stream_encoder_->SetSource(&source,
2539 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002540 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2541 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002542 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002543
2544 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002545 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002546 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2548 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2549 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2550
2551 // Trigger adapt down for same input resolution, expect no change.
2552 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2553 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002554 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002555 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2557 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2558
2559 // Trigger adapt down for larger input resolution, expect no change.
2560 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2561 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002562 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002563 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2565 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2566
mflodmancc3d4422017-08-03 08:27:51 -07002567 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002568}
2569
mflodmancc3d4422017-08-03 08:27:51 -07002570TEST_F(VideoStreamEncoderTest,
2571 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002572 const int kWidth = 1280;
2573 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002574 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002575 DataRate::BitsPerSec(kTargetBitrateBps),
2576 DataRate::BitsPerSec(kTargetBitrateBps),
2577 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002578
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002579 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002580 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002581 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002582 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002583
2584 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002585 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002586 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002587 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2588 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2589
2590 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002591 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002592 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002593 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2594 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2595
mflodmancc3d4422017-08-03 08:27:51 -07002596 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002597}
2598
mflodmancc3d4422017-08-03 08:27:51 -07002599TEST_F(VideoStreamEncoderTest,
2600 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002601 const int kWidth = 1280;
2602 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002603 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002604 DataRate::BitsPerSec(kTargetBitrateBps),
2605 DataRate::BitsPerSec(kTargetBitrateBps),
2606 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002607
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002608 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002609 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002610 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002611 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002612
2613 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002614 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002615 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002616 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002617 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2618
2619 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002620 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002621 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002622 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002623 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2624
mflodmancc3d4422017-08-03 08:27:51 -07002625 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002626}
2627
mflodmancc3d4422017-08-03 08:27:51 -07002628TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002629 const int kWidth = 1280;
2630 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002631 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002632 DataRate::BitsPerSec(kTargetBitrateBps),
2633 DataRate::BitsPerSec(kTargetBitrateBps),
2634 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002635
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002636 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002637 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002638 video_stream_encoder_->SetSource(&source,
2639 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002640
2641 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2642 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002643 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002644 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2645 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2646 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2647
2648 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002649 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002650 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2654
mflodmancc3d4422017-08-03 08:27:51 -07002655 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002656}
2657
mflodmancc3d4422017-08-03 08:27:51 -07002658TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002659 const int kWidth = 1280;
2660 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002661 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002662 DataRate::BitsPerSec(kTargetBitrateBps),
2663 DataRate::BitsPerSec(kTargetBitrateBps),
2664 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002665
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002666 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002667 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002668 video_stream_encoder_->SetSource(&source,
2669 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002670
2671 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2672 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002673 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2675 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2676 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2677
2678 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002679 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002680 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002681 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2682 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2683 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2684
mflodmancc3d4422017-08-03 08:27:51 -07002685 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002686}
2687
mflodmancc3d4422017-08-03 08:27:51 -07002688TEST_F(VideoStreamEncoderTest,
2689 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002690 const int kWidth = 1280;
2691 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002692 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002693 DataRate::BitsPerSec(kTargetBitrateBps),
2694 DataRate::BitsPerSec(kTargetBitrateBps),
2695 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002696
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002697 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002698 AdaptingFrameForwarder source;
2699 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002700 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002701 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002702
2703 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002704 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002705 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002706 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2708
2709 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002710 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002711 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002712 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002713 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002714 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2715 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2716
2717 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002718 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002719 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002720 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2721 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2722 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2723
mflodmancc3d4422017-08-03 08:27:51 -07002724 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002725}
2726
mflodmancc3d4422017-08-03 08:27:51 -07002727TEST_F(VideoStreamEncoderTest,
2728 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002729 const int kWidth = 1280;
2730 const int kHeight = 720;
2731 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002732 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002733 DataRate::BitsPerSec(kTargetBitrateBps),
2734 DataRate::BitsPerSec(kTargetBitrateBps),
2735 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002736
2737 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2738 stats.input_frame_rate = kInputFps;
2739 stats_proxy_->SetMockStats(stats);
2740
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002741 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002742 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2743 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002744 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002745
2746 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002747 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002748 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2749 sink_.WaitForEncodedFrame(2);
2750 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2751
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002752 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002753 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002754 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002755 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002756 // Give the encoder queue time to process the change in degradation preference
2757 // by waiting for an encoded frame.
2758 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2759 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002760 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002761
2762 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002763 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002764 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2765 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002766 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2767
2768 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002769 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002770 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002771
mflodmancc3d4422017-08-03 08:27:51 -07002772 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002773}
2774
mflodmancc3d4422017-08-03 08:27:51 -07002775TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002776 const int kWidth = 1280;
2777 const int kHeight = 720;
2778 const size_t kNumFrames = 10;
2779
Erik Språng4c6ca302019-04-08 15:14:01 +02002780 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002781 DataRate::BitsPerSec(kTargetBitrateBps),
2782 DataRate::BitsPerSec(kTargetBitrateBps),
2783 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002784
asaperssond0de2952017-04-21 01:47:31 -07002785 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002786 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002787 video_source_.set_adaptation_enabled(true);
2788
2789 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2790 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2791
2792 int downscales = 0;
2793 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002794 video_source_.IncomingCapturedFrame(
2795 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2796 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002797
asaperssonfab67072017-04-04 05:51:49 -07002798 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002799 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002800 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002801 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002802
2803 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2804 ++downscales;
2805
2806 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2807 EXPECT_EQ(downscales,
2808 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2809 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002810 }
mflodmancc3d4422017-08-03 08:27:51 -07002811 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002812}
2813
mflodmancc3d4422017-08-03 08:27:51 -07002814TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002815 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2816 const int kWidth = 1280;
2817 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002818 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002819 DataRate::BitsPerSec(kTargetBitrateBps),
2820 DataRate::BitsPerSec(kTargetBitrateBps),
2821 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002822
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002823 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002824 AdaptingFrameForwarder source;
2825 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002826 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002827 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002828
Åsa Persson8c1bf952018-09-13 10:42:19 +02002829 int64_t timestamp_ms = kFrameIntervalMs;
2830 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002831 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002832 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002833 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2834 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2835
2836 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002838 timestamp_ms += kFrameIntervalMs;
2839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2840 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002841 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002842 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2843 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2844
2845 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02002846 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002847 timestamp_ms += kFrameIntervalMs;
2848 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002849 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002850 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002851 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2852 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2853
2854 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002855 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002856 timestamp_ms += kFrameIntervalMs;
2857 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2858 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002859 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002860 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2861 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2862
2863 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02002864 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002865 timestamp_ms += kFrameIntervalMs;
2866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002867 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002868 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002869 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2870 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2871
mflodmancc3d4422017-08-03 08:27:51 -07002872 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002873}
2874
mflodmancc3d4422017-08-03 08:27:51 -07002875TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002876 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2877 const int kWidth = 1280;
2878 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002879 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002880 DataRate::BitsPerSec(kTargetBitrateBps),
2881 DataRate::BitsPerSec(kTargetBitrateBps),
2882 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002883
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002884 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002885 AdaptingFrameForwarder source;
2886 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002887 video_stream_encoder_->SetSource(&source,
2888 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002889
Åsa Persson8c1bf952018-09-13 10:42:19 +02002890 int64_t timestamp_ms = kFrameIntervalMs;
2891 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002892 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002893 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2895 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2896
2897 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002898 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002899 timestamp_ms += kFrameIntervalMs;
2900 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2901 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002902 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2903 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2904 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2905
2906 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002907 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002908 timestamp_ms += kFrameIntervalMs;
2909 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002910 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002911 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002912 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2913 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2914
2915 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002916 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002917 timestamp_ms += kFrameIntervalMs;
2918 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2919 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002920 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2921 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2922 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2923
2924 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002925 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002926 timestamp_ms += kFrameIntervalMs;
2927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002928 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002929 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002930 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2931 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2932
mflodmancc3d4422017-08-03 08:27:51 -07002933 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002934}
2935
Sergey Silkin41c650b2019-10-14 13:12:19 +02002936TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2937 fake_encoder_.SetResolutionBitrateLimits(
2938 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2939
2940 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002941 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2942 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2943 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2944 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002945
2946 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2947 AdaptingFrameForwarder source;
2948 source.set_adaptation_enabled(true);
2949 video_stream_encoder_->SetSource(
2950 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2951
2952 // Insert 720p frame.
2953 int64_t timestamp_ms = kFrameIntervalMs;
2954 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2955 WaitForEncodedFrame(1280, 720);
2956
2957 // Reduce bitrate and trigger adapt down.
2958 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002959 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2960 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2961 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2962 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002963 video_stream_encoder_->TriggerQualityLow();
2964
2965 // Insert 720p frame. It should be downscaled and encoded.
2966 timestamp_ms += kFrameIntervalMs;
2967 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2968 WaitForEncodedFrame(960, 540);
2969
2970 // Trigger adapt up. Higher resolution should not be requested duo to lack
2971 // of bitrate.
2972 video_stream_encoder_->TriggerQualityHigh();
2973 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2974
2975 // Increase bitrate.
2976 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002977 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2978 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2979 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2980 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002981
2982 // Trigger adapt up. Higher resolution should be requested.
2983 video_stream_encoder_->TriggerQualityHigh();
2984 VerifyFpsMaxResolutionMax(source.sink_wants());
2985
2986 video_stream_encoder_->Stop();
2987}
2988
2989TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2990 fake_encoder_.SetResolutionBitrateLimits(
2991 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2992
2993 // Set bitrate equal to min bitrate of 540p.
2994 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002995 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2996 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2997 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2998 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002999
3000 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3001 AdaptingFrameForwarder source;
3002 source.set_adaptation_enabled(true);
3003 video_stream_encoder_->SetSource(
3004 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3005
3006 // Insert 720p frame. It should be dropped and lower resolution should be
3007 // requested.
3008 int64_t timestamp_ms = kFrameIntervalMs;
3009 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3010 ExpectDroppedFrame();
3011 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3012
3013 // Insert 720p frame. It should be downscaled and encoded.
3014 timestamp_ms += kFrameIntervalMs;
3015 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3016 WaitForEncodedFrame(960, 540);
3017
3018 video_stream_encoder_->Stop();
3019}
3020
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003021class BalancedDegradationTest : public VideoStreamEncoderTest {
3022 protected:
3023 void SetupTest() {
3024 // Reset encoder for field trials to take effect.
3025 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003026 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003027
3028 // Enable BALANCED preference.
3029 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003030 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3031 }
3032
3033 void OnBitrateUpdated(int bitrate_bps) {
Ying Wang9b881ab2020-02-07 14:29:32 +01003034 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003035 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3036 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003037 }
3038
Åsa Persson45b176f2019-09-30 11:19:05 +02003039 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003040 timestamp_ms_ += kFrameIntervalMs;
3041 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003042 }
3043
3044 void InsertFrameAndWaitForEncoded() {
3045 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003046 sink_.WaitForEncodedFrame(timestamp_ms_);
3047 }
3048
3049 const int kWidth = 640; // pixels:640x360=230400
3050 const int kHeight = 360;
3051 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3052 int64_t timestamp_ms_ = 0;
3053 AdaptingFrameForwarder source_;
3054};
3055
3056TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3057 test::ScopedFieldTrials field_trials(
3058 "WebRTC-Video-BalancedDegradationSettings/"
3059 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3060 SetupTest();
3061
3062 // Force input frame rate.
3063 const int kInputFps = 24;
3064 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3065 stats.input_frame_rate = kInputFps;
3066 stats_proxy_->SetMockStats(stats);
3067
Åsa Persson45b176f2019-09-30 11:19:05 +02003068 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003069 VerifyFpsMaxResolutionMax(source_.sink_wants());
3070
3071 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003072 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3073 // QP samples.
3074 EXPECT_FALSE(
3075 video_stream_encoder_
3076 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003077 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3078
3079 video_stream_encoder_->Stop();
3080}
3081
3082TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3083 test::ScopedFieldTrials field_trials(
3084 "WebRTC-Video-BalancedDegradationSettings/"
3085 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3086 SetupTest();
3087
3088 // Force input frame rate.
3089 const int kInputFps = 25;
3090 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3091 stats.input_frame_rate = kInputFps;
3092 stats_proxy_->SetMockStats(stats);
3093
Åsa Persson45b176f2019-09-30 11:19:05 +02003094 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003095 VerifyFpsMaxResolutionMax(source_.sink_wants());
3096
3097 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003098 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3099 // samples.
3100 EXPECT_TRUE(
3101 video_stream_encoder_
3102 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003103 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3104
3105 video_stream_encoder_->Stop();
3106}
3107
3108TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3109 test::ScopedFieldTrials field_trials(
3110 "WebRTC-Video-BalancedDegradationSettings/"
3111 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3112 SetupTest();
3113
3114 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3115
Åsa Persson45b176f2019-09-30 11:19:05 +02003116 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003117 VerifyFpsMaxResolutionMax(source_.sink_wants());
3118
3119 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3120 video_stream_encoder_->TriggerQualityLow();
3121 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3122
3123 video_stream_encoder_->Stop();
3124}
3125
Åsa Perssonccfb3402019-09-25 15:13:04 +02003126TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003127 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003128 "WebRTC-Video-BalancedDegradationSettings/"
3129 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003130 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003131
Åsa Persson1b247f12019-08-14 17:26:39 +02003132 const int kMinBitrateBps = 425000;
3133 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003134 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003135
Åsa Persson45b176f2019-09-30 11:19:05 +02003136 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003137 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003138 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3139
3140 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3141 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003142 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003143 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003144 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3145
3146 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3147 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003148 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003149 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003150 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3151
Åsa Persson30ab0152019-08-27 12:22:33 +02003152 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3153 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003154 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003155 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3156 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003157 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3158
3159 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003160 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003161 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003162 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003163
Åsa Persson30ab0152019-08-27 12:22:33 +02003164 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003165 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003166 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003167 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003168 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003169 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3170
3171 video_stream_encoder_->Stop();
3172}
3173
Åsa Perssonccfb3402019-09-25 15:13:04 +02003174TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003175 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3176 test::ScopedFieldTrials field_trials(
3177 "WebRTC-Video-BalancedDegradationSettings/"
3178 "pixels:57600|129600|230400,fps:7|24|24/");
3179 SetupTest();
3180 OnBitrateUpdated(kLowTargetBitrateBps);
3181
3182 VerifyNoLimitation(source_.sink_wants());
3183
3184 // Insert frame, expect scaled down:
3185 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3186 InsertFrame();
3187 EXPECT_FALSE(WaitForFrame(1000));
3188 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3189 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3190
3191 // Insert frame, expect scaled down:
3192 // resolution (320x180@24fps).
3193 InsertFrame();
3194 EXPECT_FALSE(WaitForFrame(1000));
3195 EXPECT_LT(source_.sink_wants().max_pixel_count,
3196 source_.last_wants().max_pixel_count);
3197 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3198
3199 // Frame should not be dropped (min pixels per frame reached).
3200 InsertFrameAndWaitForEncoded();
3201
3202 video_stream_encoder_->Stop();
3203}
3204
3205TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003206 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003207 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003208 "WebRTC-Video-BalancedDegradationSettings/"
3209 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003210 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003211
Åsa Persson30ab0152019-08-27 12:22:33 +02003212 const int kResolutionMinBitrateBps = 435000;
3213 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003214 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003215
Åsa Persson45b176f2019-09-30 11:19:05 +02003216 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003217 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003218 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3219
3220 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3221 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003222 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003223 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003224 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3225
3226 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3227 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003228 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003229 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003230 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3231
3232 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3233 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003234 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003235 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003236 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3237
Åsa Persson30ab0152019-08-27 12:22:33 +02003238 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3239 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003240 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003241 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003242 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3243
3244 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3245 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003246 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003247 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3248
3249 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003250 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003251 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003252 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003253 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003254 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3255
3256 video_stream_encoder_->Stop();
3257}
3258
Åsa Perssonccfb3402019-09-25 15:13:04 +02003259TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003260 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003261 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003262 "WebRTC-Video-BalancedDegradationSettings/"
3263 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003264 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003265
Åsa Persson30ab0152019-08-27 12:22:33 +02003266 const int kMinBitrateBps = 425000;
3267 const int kTooLowMinBitrateBps = 424000;
3268 const int kResolutionMinBitrateBps = 435000;
3269 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003270 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003271
Åsa Persson45b176f2019-09-30 11:19:05 +02003272 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003273 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003274 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3275
3276 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3277 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003278 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003279 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003280 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3281
3282 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3283 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003284 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003285 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003286 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3287
3288 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3289 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003290 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003291 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003292 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3293
3294 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3295 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003296 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003297 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3298
3299 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003300 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003301 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003302 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003303 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003304 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3305
3306 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003307 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003308 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003309 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003310 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3311
3312 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003313 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003314 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003315 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003316 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003317 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3318
Åsa Persson1b247f12019-08-14 17:26:39 +02003319 video_stream_encoder_->Stop();
3320}
3321
mflodmancc3d4422017-08-03 08:27:51 -07003322TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003323 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3324 const int kWidth = 1280;
3325 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02003326 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003327 DataRate::BitsPerSec(kTargetBitrateBps),
3328 DataRate::BitsPerSec(kTargetBitrateBps),
3329 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003330
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003331 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003332 AdaptingFrameForwarder source;
3333 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003334 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003335 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003336
Åsa Persson8c1bf952018-09-13 10:42:19 +02003337 int64_t timestamp_ms = kFrameIntervalMs;
3338 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003339 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003340 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003341 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3342 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3343 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3344 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3345
3346 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003347 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003348 timestamp_ms += kFrameIntervalMs;
3349 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3350 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003351 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003352 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3353 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3354 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3355 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3356
3357 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003358 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003359 timestamp_ms += kFrameIntervalMs;
3360 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3361 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003362 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003363 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3364 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3365 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3366 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3367
Jonathan Yubc771b72017-12-08 17:04:29 -08003368 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003369 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003370 timestamp_ms += kFrameIntervalMs;
3371 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3372 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003373 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003374 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3375 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003376 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003377 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3378
Jonathan Yubc771b72017-12-08 17:04:29 -08003379 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003380 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003381 timestamp_ms += kFrameIntervalMs;
3382 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3383 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003384 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003385 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003386 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3387 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3388 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3389 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3390
Jonathan Yubc771b72017-12-08 17:04:29 -08003391 // Trigger quality adapt down, expect no change (min resolution reached).
3392 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003393 timestamp_ms += kFrameIntervalMs;
3394 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3395 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003396 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3397 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3398 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3399 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3400 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3401
3402 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003403 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003404 timestamp_ms += kFrameIntervalMs;
3405 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3406 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003407 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003408 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3410 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3411 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3412
3413 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003414 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003415 timestamp_ms += kFrameIntervalMs;
3416 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3417 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003418 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3419 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3420 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3422 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003425 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003426 timestamp_ms += kFrameIntervalMs;
3427 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3428 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003429 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003430 last_wants = source.sink_wants();
3431 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3432 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003433 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003434 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
3436 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003437 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003438 timestamp_ms += kFrameIntervalMs;
3439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3440 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003441 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3443 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003444 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003445 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3446
3447 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003448 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003449 timestamp_ms += kFrameIntervalMs;
3450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003451 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003452 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003453 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003454 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003456 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003457 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003458
mflodmancc3d4422017-08-03 08:27:51 -07003459 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003460}
3461
mflodmancc3d4422017-08-03 08:27:51 -07003462TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003463 const int kWidth = 640;
3464 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003465
Erik Språng4c6ca302019-04-08 15:14:01 +02003466 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003467 DataRate::BitsPerSec(kTargetBitrateBps),
3468 DataRate::BitsPerSec(kTargetBitrateBps),
3469 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003470
perkj803d97f2016-11-01 11:45:46 -07003471 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003472 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003473 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003474 }
3475
mflodmancc3d4422017-08-03 08:27:51 -07003476 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003477 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003478 video_source_.IncomingCapturedFrame(CreateFrame(
3479 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003480 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003481 }
3482
mflodmancc3d4422017-08-03 08:27:51 -07003483 video_stream_encoder_->Stop();
3484 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003485 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003486
Ying Wangef3998f2019-12-09 13:06:53 +01003487 EXPECT_METRIC_EQ(
3488 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3489 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003490 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3491}
3492
mflodmancc3d4422017-08-03 08:27:51 -07003493TEST_F(VideoStreamEncoderTest,
3494 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003495 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003496 DataRate::BitsPerSec(kTargetBitrateBps),
3497 DataRate::BitsPerSec(kTargetBitrateBps),
3498 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003499 const int kWidth = 640;
3500 const int kHeight = 360;
3501
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003502 video_stream_encoder_->SetSource(&video_source_,
3503 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003504
3505 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3506 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003507 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003508 }
3509
mflodmancc3d4422017-08-03 08:27:51 -07003510 video_stream_encoder_->Stop();
3511 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003512 stats_proxy_.reset();
3513
3514 EXPECT_EQ(0,
3515 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3516}
3517
mflodmancc3d4422017-08-03 08:27:51 -07003518TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003519 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003520 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003521
3522 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003523 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003524 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003525 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3526 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003527
sprang57c2fff2017-01-16 06:24:02 -08003528 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003529 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003530 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003531 DataRate::BitsPerSec(kLowTargetBitrateBps),
3532 DataRate::BitsPerSec(kLowTargetBitrateBps),
3533 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003534
sprang57c2fff2017-01-16 06:24:02 -08003535 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003536 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3537 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003538 VideoBitrateAllocation bitrate_allocation =
3539 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003540 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003541 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003542 // TODO(srte): The use of millisecs here looks like an error, but the tests
3543 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003544 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003545
3546 // Not called on second frame.
3547 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3548 .Times(0);
3549 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003550 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3551 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003552 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003553
3554 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003555 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3556 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003557 const int64_t start_time_ms = rtc::TimeMillis();
3558 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3559 video_source_.IncomingCapturedFrame(
3560 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3561 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003562 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003563 }
3564
3565 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003566 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003567
mflodmancc3d4422017-08-03 08:27:51 -07003568 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003569}
3570
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003571TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3572 // 2 TLs configured, temporal layers supported by encoder.
3573 const int kNumTemporalLayers = 2;
3574 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3575 fake_encoder_.SetTemporalLayersSupported(0, true);
3576
3577 // Bitrate allocated across temporal layers.
3578 const int kTl0Bps = kTargetBitrateBps *
3579 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003580 kNumTemporalLayers, /*temporal_id*/ 0,
3581 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003582 const int kTl1Bps = kTargetBitrateBps *
3583 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003584 kNumTemporalLayers, /*temporal_id*/ 1,
3585 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003586 VideoBitrateAllocation expected_bitrate;
3587 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3588 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3589
3590 VerifyAllocatedBitrate(expected_bitrate);
3591 video_stream_encoder_->Stop();
3592}
3593
3594TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3595 // 2 TLs configured, temporal layers not supported by encoder.
3596 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3597 fake_encoder_.SetTemporalLayersSupported(0, false);
3598
3599 // Temporal layers not supported by the encoder.
3600 // Total bitrate should be at ti:0.
3601 VideoBitrateAllocation expected_bitrate;
3602 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3603
3604 VerifyAllocatedBitrate(expected_bitrate);
3605 video_stream_encoder_->Stop();
3606}
3607
3608TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3609 // 2 TLs configured, temporal layers only supported for first stream.
3610 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3611 fake_encoder_.SetTemporalLayersSupported(0, true);
3612 fake_encoder_.SetTemporalLayersSupported(1, false);
3613
3614 const int kS0Bps = 150000;
3615 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003616 kS0Bps *
3617 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3618 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003619 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003620 kS0Bps *
3621 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3622 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003623 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3624 // Temporal layers not supported by si:1.
3625 VideoBitrateAllocation expected_bitrate;
3626 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3627 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3628 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3629
3630 VerifyAllocatedBitrate(expected_bitrate);
3631 video_stream_encoder_->Stop();
3632}
3633
Niels Möller7dc26b72017-12-06 10:27:48 +01003634TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3635 const int kFrameWidth = 1280;
3636 const int kFrameHeight = 720;
3637 const int kFramerate = 24;
3638
Erik Språng4c6ca302019-04-08 15:14:01 +02003639 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003640 DataRate::BitsPerSec(kTargetBitrateBps),
3641 DataRate::BitsPerSec(kTargetBitrateBps),
3642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003643 test::FrameForwarder source;
3644 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003645 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003646
3647 // Insert a single frame, triggering initial configuration.
3648 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3649 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3650
3651 EXPECT_EQ(
3652 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3653 kDefaultFramerate);
3654
3655 // Trigger reconfigure encoder (without resetting the entire instance).
3656 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003657 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003658 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3659 video_encoder_config.number_of_streams = 1;
3660 video_encoder_config.video_stream_factory =
3661 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3662 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003663 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003664 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3665
3666 // Detector should be updated with fps limit from codec config.
3667 EXPECT_EQ(
3668 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3669 kFramerate);
3670
3671 // Trigger overuse, max framerate should be reduced.
3672 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3673 stats.input_frame_rate = kFramerate;
3674 stats_proxy_->SetMockStats(stats);
3675 video_stream_encoder_->TriggerCpuOveruse();
3676 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3677 int adapted_framerate =
3678 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3679 EXPECT_LT(adapted_framerate, kFramerate);
3680
3681 // Trigger underuse, max framerate should go back to codec configured fps.
3682 // Set extra low fps, to make sure it's actually reset, not just incremented.
3683 stats = stats_proxy_->GetStats();
3684 stats.input_frame_rate = adapted_framerate / 2;
3685 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003686 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003687 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3688 EXPECT_EQ(
3689 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3690 kFramerate);
3691
3692 video_stream_encoder_->Stop();
3693}
3694
3695TEST_F(VideoStreamEncoderTest,
3696 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3697 const int kFrameWidth = 1280;
3698 const int kFrameHeight = 720;
3699 const int kLowFramerate = 15;
3700 const int kHighFramerate = 25;
3701
Erik Språng4c6ca302019-04-08 15:14:01 +02003702 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003703 DataRate::BitsPerSec(kTargetBitrateBps),
3704 DataRate::BitsPerSec(kTargetBitrateBps),
3705 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003706 test::FrameForwarder source;
3707 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003708 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003709
3710 // Trigger initial configuration.
3711 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003712 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003713 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3714 video_encoder_config.number_of_streams = 1;
3715 video_encoder_config.video_stream_factory =
3716 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3717 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3718 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003719 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003720 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3721
3722 EXPECT_EQ(
3723 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3724 kLowFramerate);
3725
3726 // Trigger overuse, max framerate should be reduced.
3727 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3728 stats.input_frame_rate = kLowFramerate;
3729 stats_proxy_->SetMockStats(stats);
3730 video_stream_encoder_->TriggerCpuOveruse();
3731 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3732 int adapted_framerate =
3733 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3734 EXPECT_LT(adapted_framerate, kLowFramerate);
3735
3736 // Reconfigure the encoder with a new (higher max framerate), max fps should
3737 // still respect the adaptation.
3738 video_encoder_config.video_stream_factory =
3739 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3740 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3741 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003742 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003743 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3744
3745 EXPECT_EQ(
3746 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3747 adapted_framerate);
3748
3749 // Trigger underuse, max framerate should go back to codec configured fps.
3750 stats = stats_proxy_->GetStats();
3751 stats.input_frame_rate = adapted_framerate;
3752 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003753 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003754 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3755 EXPECT_EQ(
3756 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3757 kHighFramerate);
3758
3759 video_stream_encoder_->Stop();
3760}
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762TEST_F(VideoStreamEncoderTest,
3763 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003764 const int kFrameWidth = 1280;
3765 const int kFrameHeight = 720;
3766 const int kFramerate = 24;
3767
Erik Språng4c6ca302019-04-08 15:14:01 +02003768 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003769 DataRate::BitsPerSec(kTargetBitrateBps),
3770 DataRate::BitsPerSec(kTargetBitrateBps),
3771 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003772 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003773 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003774 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003775
3776 // Trigger initial configuration.
3777 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003778 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003779 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3780 video_encoder_config.number_of_streams = 1;
3781 video_encoder_config.video_stream_factory =
3782 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3783 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003784 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003785 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003786 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003787
Niels Möller7dc26b72017-12-06 10:27:48 +01003788 EXPECT_EQ(
3789 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3790 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003791
3792 // Trigger overuse, max framerate should be reduced.
3793 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3794 stats.input_frame_rate = kFramerate;
3795 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003796 video_stream_encoder_->TriggerCpuOveruse();
3797 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003798 int adapted_framerate =
3799 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003800 EXPECT_LT(adapted_framerate, kFramerate);
3801
3802 // Change degradation preference to not enable framerate scaling. Target
3803 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003804 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003805 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003806 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003807 EXPECT_EQ(
3808 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3809 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003810
mflodmancc3d4422017-08-03 08:27:51 -07003811 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003812}
3813
mflodmancc3d4422017-08-03 08:27:51 -07003814TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003815 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003816 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003817 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3818 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3819 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003820 const int kWidth = 640;
3821 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003822
asaperssonfab67072017-04-04 05:51:49 -07003823 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003824
3825 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003826 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003827
3828 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003829 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003830
sprangc5d62e22017-04-02 23:53:04 -07003831 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003832
asaperssonfab67072017-04-04 05:51:49 -07003833 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003834 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003835 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003836
3837 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003838 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003839
sprangc5d62e22017-04-02 23:53:04 -07003840 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003841
mflodmancc3d4422017-08-03 08:27:51 -07003842 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003843}
3844
mflodmancc3d4422017-08-03 08:27:51 -07003845TEST_F(VideoStreamEncoderTest,
3846 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003847 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003848 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003849 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3850 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3851 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003852 const int kWidth = 640;
3853 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003854
3855 // We expect the n initial frames to get dropped.
3856 int i;
3857 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003858 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003859 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003860 }
3861 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003862 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003863 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003864
3865 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003866 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003867
mflodmancc3d4422017-08-03 08:27:51 -07003868 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003869}
3870
mflodmancc3d4422017-08-03 08:27:51 -07003871TEST_F(VideoStreamEncoderTest,
3872 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003873 const int kWidth = 640;
3874 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003875 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003876 DataRate::BitsPerSec(kLowTargetBitrateBps),
3877 DataRate::BitsPerSec(kLowTargetBitrateBps),
3878 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003879
3880 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003881 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003882 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003883
asaperssonfab67072017-04-04 05:51:49 -07003884 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003885 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003886 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003887
mflodmancc3d4422017-08-03 08:27:51 -07003888 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003889}
3890
mflodmancc3d4422017-08-03 08:27:51 -07003891TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003892 const int kWidth = 640;
3893 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003894 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003895
3896 VideoEncoderConfig video_encoder_config;
3897 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3898 // Make format different, to force recreation of encoder.
3899 video_encoder_config.video_format.parameters["foo"] = "foo";
3900 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003901 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003902 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003903 DataRate::BitsPerSec(kLowTargetBitrateBps),
3904 DataRate::BitsPerSec(kLowTargetBitrateBps),
3905 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003906
kthelgasonb83797b2017-02-14 11:57:25 -08003907 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003908 video_stream_encoder_->SetSource(&video_source_,
3909 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003910
asaperssonfab67072017-04-04 05:51:49 -07003911 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003912 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003913 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003914
mflodmancc3d4422017-08-03 08:27:51 -07003915 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003916 fake_encoder_.SetQualityScaling(true);
3917}
3918
Åsa Persson139f4dc2019-08-02 09:29:58 +02003919TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3920 webrtc::test::ScopedFieldTrials field_trials(
3921 "WebRTC-Video-QualityScalerSettings/"
3922 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3923 // Reset encoder for field trials to take effect.
3924 ConfigureEncoder(video_encoder_config_.Copy());
3925 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3926 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3927 const int kWidth = 640;
3928 const int kHeight = 360;
3929
3930 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003931 DataRate::BitsPerSec(kTargetBitrateBps),
3932 DataRate::BitsPerSec(kTargetBitrateBps),
3933 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003934 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3935 // Frame should not be dropped.
3936 WaitForEncodedFrame(1);
3937
3938 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003939 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3940 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3941 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003942 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3943 // Frame should not be dropped.
3944 WaitForEncodedFrame(2);
3945
3946 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003947 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3948 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3949 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003950 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3951 // Expect to drop this frame, the wait should time out.
3952 ExpectDroppedFrame();
3953
3954 // Expect the sink_wants to specify a scaled frame.
3955 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3956 video_stream_encoder_->Stop();
3957}
3958
Åsa Perssone644a032019-11-08 15:56:00 +01003959TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3960 webrtc::test::ScopedFieldTrials field_trials(
3961 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3962
3963 // Reset encoder for field trials to take effect.
3964 VideoEncoderConfig config = video_encoder_config_.Copy();
3965 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02003966 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01003967 ConfigureEncoder(std::move(config));
3968 fake_encoder_.SetQp(kQpLow);
3969
3970 // Enable MAINTAIN_FRAMERATE preference.
3971 AdaptingFrameForwarder source;
3972 source.set_adaptation_enabled(true);
3973 video_stream_encoder_->SetSource(&source,
3974 DegradationPreference::MAINTAIN_FRAMERATE);
3975
3976 // Start at low bitrate.
3977 const int kLowBitrateBps = 200000;
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003978 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(kLowBitrateBps),
3979 DataRate::BitsPerSec(kLowBitrateBps),
3980 DataRate::BitsPerSec(kLowBitrateBps),
3981 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003982
3983 // Expect first frame to be dropped and resolution to be limited.
3984 const int kWidth = 1280;
3985 const int kHeight = 720;
3986 const int64_t kFrameIntervalMs = 100;
3987 int64_t timestamp_ms = kFrameIntervalMs;
3988 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3989 ExpectDroppedFrame();
3990 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3991
3992 // Increase bitrate to encoder max.
Evan Shrubsoledff79252020-04-16 11:34:32 +02003993 video_stream_encoder_->OnBitrateUpdated(max_bitrate, max_bitrate, max_bitrate,
3994 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003995
3996 // Insert frames and advance |min_duration_ms|.
3997 for (size_t i = 1; i <= 10; i++) {
3998 timestamp_ms += kFrameIntervalMs;
3999 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4000 WaitForEncodedFrame(timestamp_ms);
4001 }
4002 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4003 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4004
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004005 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004006
4007 // Insert frame should trigger high BW and release quality limitation.
4008 timestamp_ms += kFrameIntervalMs;
4009 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4010 WaitForEncodedFrame(timestamp_ms);
4011 VerifyFpsMaxResolutionMax(source.sink_wants());
4012
4013 // Frame should not be adapted.
4014 timestamp_ms += kFrameIntervalMs;
4015 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4016 WaitForEncodedFrame(kWidth, kHeight);
4017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4018
4019 video_stream_encoder_->Stop();
4020}
4021
mflodmancc3d4422017-08-03 08:27:51 -07004022TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004023 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4024 const int kTooSmallWidth = 10;
4025 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02004026 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004027 DataRate::BitsPerSec(kTargetBitrateBps),
4028 DataRate::BitsPerSec(kTargetBitrateBps),
4029 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004030
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004031 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004032 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004033 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004034 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004035 VerifyNoLimitation(source.sink_wants());
4036 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4037
4038 // Trigger adapt down, too small frame, expect no change.
4039 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004040 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004041 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004042 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07004043 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4044 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4045
mflodmancc3d4422017-08-03 08:27:51 -07004046 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004047}
4048
mflodmancc3d4422017-08-03 08:27:51 -07004049TEST_F(VideoStreamEncoderTest,
4050 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004051 const int kTooSmallWidth = 10;
4052 const int kTooSmallHeight = 10;
4053 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02004054 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004055 DataRate::BitsPerSec(kTargetBitrateBps),
4056 DataRate::BitsPerSec(kTargetBitrateBps),
4057 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004058
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004059 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004060 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004061 video_stream_encoder_->SetSource(&source,
4062 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004063 VerifyNoLimitation(source.sink_wants());
4064 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4065 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4066
4067 // Trigger adapt down, expect limited framerate.
4068 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004069 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004070 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004071 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4072 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4073 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4074 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4075
4076 // Trigger adapt down, too small frame, expect no change.
4077 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004078 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004079 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004080 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4081 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4082 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4083 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4084
mflodmancc3d4422017-08-03 08:27:51 -07004085 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004086}
4087
mflodmancc3d4422017-08-03 08:27:51 -07004088TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004089 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02004090 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004091 DataRate::BitsPerSec(kTargetBitrateBps),
4092 DataRate::BitsPerSec(kTargetBitrateBps),
4093 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004094 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004095 const int kFrameWidth = 1280;
4096 const int kFrameHeight = 720;
4097 video_source_.IncomingCapturedFrame(
4098 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004099 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004100 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004101}
4102
sprangb1ca0732017-02-01 08:38:12 -08004103// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004104TEST_F(VideoStreamEncoderTest,
4105 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004106 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004107 DataRate::BitsPerSec(kTargetBitrateBps),
4108 DataRate::BitsPerSec(kTargetBitrateBps),
4109 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004110
4111 const int kFrameWidth = 1280;
4112 const int kFrameHeight = 720;
4113 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004114 // requested by
4115 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004116 video_source_.set_adaptation_enabled(true);
4117
4118 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004119 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004120 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004121
4122 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004123 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004124 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004125 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004126 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004127
asaperssonfab67072017-04-04 05:51:49 -07004128 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004129 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004130 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004131 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004132 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004133
mflodmancc3d4422017-08-03 08:27:51 -07004134 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004135}
sprangfe627f32017-03-29 08:24:59 -07004136
mflodmancc3d4422017-08-03 08:27:51 -07004137TEST_F(VideoStreamEncoderTest,
4138 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004139 const int kFrameWidth = 1280;
4140 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004141
Erik Språng4c6ca302019-04-08 15:14:01 +02004142 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004143 DataRate::BitsPerSec(kTargetBitrateBps),
4144 DataRate::BitsPerSec(kTargetBitrateBps),
4145 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004146 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004147 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004148 video_source_.set_adaptation_enabled(true);
4149
sprang4847ae62017-06-27 07:06:52 -07004150 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004151
4152 video_source_.IncomingCapturedFrame(
4153 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004154 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004155
4156 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004157 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004158
4159 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004160 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004161 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004162 video_source_.IncomingCapturedFrame(
4163 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004164 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004165 }
4166
4167 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004168 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004169 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004170 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004171 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004172 video_source_.IncomingCapturedFrame(
4173 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004174 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004175 ++num_frames_dropped;
4176 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004177 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004178 }
4179 }
4180
sprang4847ae62017-06-27 07:06:52 -07004181 // Add some slack to account for frames dropped by the frame dropper.
4182 const int kErrorMargin = 1;
4183 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004184 kErrorMargin);
4185
4186 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004187 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004188 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004189 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004190 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004191 video_source_.IncomingCapturedFrame(
4192 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004193 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004194 ++num_frames_dropped;
4195 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004196 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004197 }
4198 }
sprang4847ae62017-06-27 07:06:52 -07004199 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004200 kErrorMargin);
4201
4202 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004203 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004204 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004205 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004206 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004207 video_source_.IncomingCapturedFrame(
4208 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004209 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004210 ++num_frames_dropped;
4211 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004212 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004213 }
4214 }
sprang4847ae62017-06-27 07:06:52 -07004215 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004216 kErrorMargin);
4217
4218 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004219 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004220 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004221 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004222 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004223 video_source_.IncomingCapturedFrame(
4224 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004225 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004226 ++num_frames_dropped;
4227 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004228 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004229 }
4230 }
4231 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4232
mflodmancc3d4422017-08-03 08:27:51 -07004233 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004234}
4235
mflodmancc3d4422017-08-03 08:27:51 -07004236TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004237 const int kFramerateFps = 5;
4238 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004239 const int kFrameWidth = 1280;
4240 const int kFrameHeight = 720;
4241
sprang4847ae62017-06-27 07:06:52 -07004242 // Reconfigure encoder with two temporal layers and screensharing, which will
4243 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004244 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004245
Erik Språng4c6ca302019-04-08 15:14:01 +02004246 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004247 DataRate::BitsPerSec(kTargetBitrateBps),
4248 DataRate::BitsPerSec(kTargetBitrateBps),
4249 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004250 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004251 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004252 video_source_.set_adaptation_enabled(true);
4253
sprang4847ae62017-06-27 07:06:52 -07004254 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004255
4256 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004257 rtc::VideoSinkWants last_wants;
4258 do {
4259 last_wants = video_source_.sink_wants();
4260
sprangc5d62e22017-04-02 23:53:04 -07004261 // Insert frames to get a new fps estimate...
4262 for (int j = 0; j < kFramerateFps; ++j) {
4263 video_source_.IncomingCapturedFrame(
4264 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004265 if (video_source_.last_sent_width()) {
4266 sink_.WaitForEncodedFrame(timestamp_ms);
4267 }
sprangc5d62e22017-04-02 23:53:04 -07004268 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004269 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004270 }
4271 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004272 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004273 } while (video_source_.sink_wants().max_framerate_fps <
4274 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004275
Jonathan Yubc771b72017-12-08 17:04:29 -08004276 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004277
mflodmancc3d4422017-08-03 08:27:51 -07004278 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004279}
asaperssonf7e294d2017-06-13 23:25:22 -07004280
mflodmancc3d4422017-08-03 08:27:51 -07004281TEST_F(VideoStreamEncoderTest,
4282 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004283 const int kWidth = 1280;
4284 const int kHeight = 720;
4285 const int64_t kFrameIntervalMs = 150;
4286 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004287 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004288 DataRate::BitsPerSec(kTargetBitrateBps),
4289 DataRate::BitsPerSec(kTargetBitrateBps),
4290 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004291
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004292 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004293 AdaptingFrameForwarder source;
4294 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004295 video_stream_encoder_->SetSource(&source,
4296 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004297 timestamp_ms += kFrameIntervalMs;
4298 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004299 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004300 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004301 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4302 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4303 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4304
4305 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004306 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004307 timestamp_ms += kFrameIntervalMs;
4308 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004309 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004310 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4311 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4312 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4313 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4314
4315 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004316 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004317 timestamp_ms += kFrameIntervalMs;
4318 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004319 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004320 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4321 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4322 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4323 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4324
4325 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004326 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004327 timestamp_ms += kFrameIntervalMs;
4328 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004329 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004330 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4331 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4332 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4333 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4334
4335 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004336 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004337 timestamp_ms += kFrameIntervalMs;
4338 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004339 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004340 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4341 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4342 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4343 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4344
4345 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004346 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004347 timestamp_ms += kFrameIntervalMs;
4348 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004349 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004350 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4351 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4352 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4353 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4354
4355 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004356 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004357 timestamp_ms += kFrameIntervalMs;
4358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004359 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004360 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4361 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4362 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4363 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4364
4365 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004366 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004367 timestamp_ms += kFrameIntervalMs;
4368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004369 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004370 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4371 rtc::VideoSinkWants last_wants = source.sink_wants();
4372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4373 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4374 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4375
4376 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004377 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004378 timestamp_ms += kFrameIntervalMs;
4379 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004380 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004381 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4382 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4383 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4384 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4385
4386 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004387 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004388 timestamp_ms += kFrameIntervalMs;
4389 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004390 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004391 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4392 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4393 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4394 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4395
4396 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004397 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004398 timestamp_ms += kFrameIntervalMs;
4399 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004400 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004401 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4402 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4403 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4404 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4405
4406 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004407 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004408 timestamp_ms += kFrameIntervalMs;
4409 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004410 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004411 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4412 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4413 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4414 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4415
4416 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004417 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004418 timestamp_ms += kFrameIntervalMs;
4419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004420 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004421 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4423 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4424 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4425
4426 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004427 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004428 timestamp_ms += kFrameIntervalMs;
4429 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004430 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004431 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4432 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4434 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4435
4436 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004437 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004438 timestamp_ms += kFrameIntervalMs;
4439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004440 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004441 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4442 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4444 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4445
Åsa Persson30ab0152019-08-27 12:22:33 +02004446 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004447 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004448 timestamp_ms += kFrameIntervalMs;
4449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004450 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004451 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004452 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4454 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4455 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4456
4457 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004458 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004459 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004460 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
mflodmancc3d4422017-08-03 08:27:51 -07004462 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004463}
4464
mflodmancc3d4422017-08-03 08:27:51 -07004465TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004466 const int kWidth = 1280;
4467 const int kHeight = 720;
4468 const int64_t kFrameIntervalMs = 150;
4469 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004470 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004471 DataRate::BitsPerSec(kTargetBitrateBps),
4472 DataRate::BitsPerSec(kTargetBitrateBps),
4473 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004474
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004475 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004476 AdaptingFrameForwarder source;
4477 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004478 video_stream_encoder_->SetSource(&source,
4479 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004480 timestamp_ms += kFrameIntervalMs;
4481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004482 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004483 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004484 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4486 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4487 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4488 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4489 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4490
4491 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004492 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004493 timestamp_ms += kFrameIntervalMs;
4494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004495 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004496 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4499 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4500 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4501 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4502 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4503
4504 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004505 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004506 timestamp_ms += kFrameIntervalMs;
4507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004508 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004509 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4510 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4511 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4512 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4513 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4514 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4515 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4516
4517 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004518 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004519 timestamp_ms += kFrameIntervalMs;
4520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004521 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004522 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4525 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4526 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4527 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4528 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4529
4530 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004531 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004532 timestamp_ms += kFrameIntervalMs;
4533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004534 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004535 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4538 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4539 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4540 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4541 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4542
4543 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004544 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004545 timestamp_ms += kFrameIntervalMs;
4546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004547 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004548 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4551 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4553 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4554 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4555
4556 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004557 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004558 timestamp_ms += kFrameIntervalMs;
4559 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004560 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004561 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004562 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4564 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4565 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4566 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4567 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4568 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4569
4570 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004571 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004572 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004573 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4574 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4575
mflodmancc3d4422017-08-03 08:27:51 -07004576 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004577}
4578
mflodmancc3d4422017-08-03 08:27:51 -07004579TEST_F(VideoStreamEncoderTest,
4580 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004581 const int kWidth = 640;
4582 const int kHeight = 360;
4583 const int kFpsLimit = 15;
4584 const int64_t kFrameIntervalMs = 150;
4585 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004586 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004587 DataRate::BitsPerSec(kTargetBitrateBps),
4588 DataRate::BitsPerSec(kTargetBitrateBps),
4589 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004590
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004591 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004592 AdaptingFrameForwarder source;
4593 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004594 video_stream_encoder_->SetSource(&source,
4595 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004596 timestamp_ms += kFrameIntervalMs;
4597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004598 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004599 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004600 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4602 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4603 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4604 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4605 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4606
4607 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004608 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004609 timestamp_ms += kFrameIntervalMs;
4610 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004611 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004612 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4613 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4614 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4617 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4618 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4619
4620 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004621 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004622 timestamp_ms += kFrameIntervalMs;
4623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004624 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004625 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4627 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4629 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4630 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4631 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4632
4633 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004634 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004635 timestamp_ms += kFrameIntervalMs;
4636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004637 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004638 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4639 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4641 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4642 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4643 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4644 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4645
4646 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004647 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004648 timestamp_ms += kFrameIntervalMs;
4649 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004650 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004651 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004652 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4654 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4655 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4656 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4657 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4658
4659 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004660 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004661 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004662 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4663 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4664
mflodmancc3d4422017-08-03 08:27:51 -07004665 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004666}
4667
mflodmancc3d4422017-08-03 08:27:51 -07004668TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004669 const int kFrameWidth = 1920;
4670 const int kFrameHeight = 1080;
4671 // 3/4 of 1920.
4672 const int kAdaptedFrameWidth = 1440;
4673 // 3/4 of 1080 rounded down to multiple of 4.
4674 const int kAdaptedFrameHeight = 808;
4675 const int kFramerate = 24;
4676
Erik Språng4c6ca302019-04-08 15:14:01 +02004677 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004678 DataRate::BitsPerSec(kTargetBitrateBps),
4679 DataRate::BitsPerSec(kTargetBitrateBps),
4680 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004681 // Trigger reconfigure encoder (without resetting the entire instance).
4682 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004683 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004684 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4685 video_encoder_config.number_of_streams = 1;
4686 video_encoder_config.video_stream_factory =
4687 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004688 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004689 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004690 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004691
4692 video_source_.set_adaptation_enabled(true);
4693
4694 video_source_.IncomingCapturedFrame(
4695 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004696 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004697
4698 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004699 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004700 video_source_.IncomingCapturedFrame(
4701 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004702 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004703
mflodmancc3d4422017-08-03 08:27:51 -07004704 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004705}
4706
mflodmancc3d4422017-08-03 08:27:51 -07004707TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004708 const int kFrameWidth = 1280;
4709 const int kFrameHeight = 720;
4710 const int kLowFps = 2;
4711 const int kHighFps = 30;
4712
Erik Språng4c6ca302019-04-08 15:14:01 +02004713 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004714 DataRate::BitsPerSec(kTargetBitrateBps),
4715 DataRate::BitsPerSec(kTargetBitrateBps),
4716 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004717
4718 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4719 max_framerate_ = kLowFps;
4720
4721 // Insert 2 seconds of 2fps video.
4722 for (int i = 0; i < kLowFps * 2; ++i) {
4723 video_source_.IncomingCapturedFrame(
4724 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4725 WaitForEncodedFrame(timestamp_ms);
4726 timestamp_ms += 1000 / kLowFps;
4727 }
4728
4729 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004730 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004731 DataRate::BitsPerSec(kTargetBitrateBps),
4732 DataRate::BitsPerSec(kTargetBitrateBps),
4733 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004734 video_source_.IncomingCapturedFrame(
4735 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4736 WaitForEncodedFrame(timestamp_ms);
4737 timestamp_ms += 1000 / kLowFps;
4738
4739 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4740
4741 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004742 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004743 const int kFrameIntervalMs = 1000 / kHighFps;
4744 max_framerate_ = kHighFps;
4745 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4746 video_source_.IncomingCapturedFrame(
4747 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4748 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4749 // be dropped if the encoder hans't been updated with the new higher target
4750 // framerate yet, causing it to overshoot the target bitrate and then
4751 // suffering the wrath of the media optimizer.
4752 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4753 timestamp_ms += kFrameIntervalMs;
4754 }
4755
4756 // Don expect correct measurement just yet, but it should be higher than
4757 // before.
4758 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4759
mflodmancc3d4422017-08-03 08:27:51 -07004760 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004761}
4762
mflodmancc3d4422017-08-03 08:27:51 -07004763TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004764 const int kFrameWidth = 1280;
4765 const int kFrameHeight = 720;
4766 const int kTargetBitrateBps = 1000000;
4767
4768 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004769 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004770 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004771 DataRate::BitsPerSec(kTargetBitrateBps),
4772 DataRate::BitsPerSec(kTargetBitrateBps),
4773 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004774 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004775
4776 // Insert a first video frame, causes another bitrate update.
4777 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4778 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4779 video_source_.IncomingCapturedFrame(
4780 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4781 WaitForEncodedFrame(timestamp_ms);
4782
4783 // Next, simulate video suspension due to pacer queue overrun.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004784 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
4785 DataRate::BitsPerSec(0),
4786 DataRate::BitsPerSec(0), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004787
4788 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004789 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004790 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004791
4792 // Bitrate observer should not be called.
4793 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4794 video_source_.IncomingCapturedFrame(
4795 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4796 ExpectDroppedFrame();
4797
mflodmancc3d4422017-08-03 08:27:51 -07004798 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004799}
ilnik6b826ef2017-06-16 06:53:48 -07004800
Niels Möller4db138e2018-04-19 09:04:13 +02004801TEST_F(VideoStreamEncoderTest,
4802 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4803 const int kFrameWidth = 1280;
4804 const int kFrameHeight = 720;
4805 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004806 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004807 DataRate::BitsPerSec(kTargetBitrateBps),
4808 DataRate::BitsPerSec(kTargetBitrateBps),
4809 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004810 video_source_.IncomingCapturedFrame(
4811 CreateFrame(1, kFrameWidth, kFrameHeight));
4812 WaitForEncodedFrame(1);
4813 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4814 .low_encode_usage_threshold_percent,
4815 default_options.low_encode_usage_threshold_percent);
4816 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4817 .high_encode_usage_threshold_percent,
4818 default_options.high_encode_usage_threshold_percent);
4819 video_stream_encoder_->Stop();
4820}
4821
4822TEST_F(VideoStreamEncoderTest,
4823 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4824 const int kFrameWidth = 1280;
4825 const int kFrameHeight = 720;
4826 CpuOveruseOptions hardware_options;
4827 hardware_options.low_encode_usage_threshold_percent = 150;
4828 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004829 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004830
Erik Språng4c6ca302019-04-08 15:14:01 +02004831 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004832 DataRate::BitsPerSec(kTargetBitrateBps),
4833 DataRate::BitsPerSec(kTargetBitrateBps),
4834 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004835 video_source_.IncomingCapturedFrame(
4836 CreateFrame(1, kFrameWidth, kFrameHeight));
4837 WaitForEncodedFrame(1);
4838 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4839 .low_encode_usage_threshold_percent,
4840 hardware_options.low_encode_usage_threshold_percent);
4841 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4842 .high_encode_usage_threshold_percent,
4843 hardware_options.high_encode_usage_threshold_percent);
4844 video_stream_encoder_->Stop();
4845}
4846
Niels Möller6bb5ab92019-01-11 11:11:10 +01004847TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4848 const int kFrameWidth = 320;
4849 const int kFrameHeight = 240;
4850 const int kFps = 30;
4851 const int kTargetBitrateBps = 120000;
4852 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4853
Erik Språng4c6ca302019-04-08 15:14:01 +02004854 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004855 DataRate::BitsPerSec(kTargetBitrateBps),
4856 DataRate::BitsPerSec(kTargetBitrateBps),
4857 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004858
4859 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4860 max_framerate_ = kFps;
4861
4862 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4863 fake_encoder_.SimulateOvershoot(1.0);
4864 int num_dropped = 0;
4865 for (int i = 0; i < kNumFramesInRun; ++i) {
4866 video_source_.IncomingCapturedFrame(
4867 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4868 // Wait up to two frame durations for a frame to arrive.
4869 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4870 ++num_dropped;
4871 }
4872 timestamp_ms += 1000 / kFps;
4873 }
4874
Erik Språnga8d48ab2019-02-08 14:17:40 +01004875 // Framerate should be measured to be near the expected target rate.
4876 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4877
4878 // Frame drops should be within 5% of expected 0%.
4879 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004880
4881 // Make encoder produce frames at double the expected bitrate during 3 seconds
4882 // of video, verify number of drops. Rate needs to be slightly changed in
4883 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004884 double overshoot_factor = 2.0;
4885 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4886 // With bitrate adjuster, when need to overshoot even more to trigger
4887 // frame dropping.
4888 overshoot_factor *= 2;
4889 }
4890 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004891 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004892 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4893 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4894 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004895 num_dropped = 0;
4896 for (int i = 0; i < kNumFramesInRun; ++i) {
4897 video_source_.IncomingCapturedFrame(
4898 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4899 // Wait up to two frame durations for a frame to arrive.
4900 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4901 ++num_dropped;
4902 }
4903 timestamp_ms += 1000 / kFps;
4904 }
4905
Erik Språng4c6ca302019-04-08 15:14:01 +02004906 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004907 DataRate::BitsPerSec(kTargetBitrateBps),
4908 DataRate::BitsPerSec(kTargetBitrateBps),
4909 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004910
4911 // Target framerate should be still be near the expected target, despite
4912 // the frame drops.
4913 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4914
4915 // Frame drops should be within 5% of expected 50%.
4916 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004917
4918 video_stream_encoder_->Stop();
4919}
4920
4921TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4922 const int kFrameWidth = 320;
4923 const int kFrameHeight = 240;
4924 const int kActualInputFps = 24;
4925 const int kTargetBitrateBps = 120000;
4926
4927 ASSERT_GT(max_framerate_, kActualInputFps);
4928
4929 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4930 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004931 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004932 DataRate::BitsPerSec(kTargetBitrateBps),
4933 DataRate::BitsPerSec(kTargetBitrateBps),
4934 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004935
4936 // Insert 3 seconds of video, with an input fps lower than configured max.
4937 for (int i = 0; i < kActualInputFps * 3; ++i) {
4938 video_source_.IncomingCapturedFrame(
4939 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4940 // Wait up to two frame durations for a frame to arrive.
4941 WaitForEncodedFrame(timestamp_ms);
4942 timestamp_ms += 1000 / kActualInputFps;
4943 }
4944
4945 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4946
4947 video_stream_encoder_->Stop();
4948}
4949
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004950TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4951 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004952 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004953 DataRate::BitsPerSec(kTargetBitrateBps),
4954 DataRate::BitsPerSec(kTargetBitrateBps),
4955 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004956
4957 fake_encoder_.BlockNextEncode();
4958 video_source_.IncomingCapturedFrame(
4959 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4960 WaitForEncodedFrame(1);
4961 // On the very first frame full update should be forced.
4962 rect = fake_encoder_.GetLastUpdateRect();
4963 EXPECT_EQ(rect.offset_x, 0);
4964 EXPECT_EQ(rect.offset_y, 0);
4965 EXPECT_EQ(rect.height, codec_height_);
4966 EXPECT_EQ(rect.width, codec_width_);
4967 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4968 // call to ContinueEncode.
4969 video_source_.IncomingCapturedFrame(
4970 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4971 ExpectDroppedFrame();
4972 video_source_.IncomingCapturedFrame(
4973 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4974 ExpectDroppedFrame();
4975 fake_encoder_.ContinueEncode();
4976 WaitForEncodedFrame(3);
4977 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4978 rect = fake_encoder_.GetLastUpdateRect();
4979 EXPECT_EQ(rect.offset_x, 1);
4980 EXPECT_EQ(rect.offset_y, 0);
4981 EXPECT_EQ(rect.width, 10);
4982 EXPECT_EQ(rect.height, 1);
4983
4984 video_source_.IncomingCapturedFrame(
4985 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4986 WaitForEncodedFrame(4);
4987 // Previous frame was encoded, so no accumulation should happen.
4988 rect = fake_encoder_.GetLastUpdateRect();
4989 EXPECT_EQ(rect.offset_x, 0);
4990 EXPECT_EQ(rect.offset_y, 0);
4991 EXPECT_EQ(rect.width, 1);
4992 EXPECT_EQ(rect.height, 1);
4993
4994 video_stream_encoder_->Stop();
4995}
4996
Erik Språngd7329ca2019-02-21 21:19:53 +01004997TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004998 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004999 DataRate::BitsPerSec(kTargetBitrateBps),
5000 DataRate::BitsPerSec(kTargetBitrateBps),
5001 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005002
5003 // First frame is always keyframe.
5004 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5005 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005006 EXPECT_THAT(
5007 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005008 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005009
5010 // Insert delta frame.
5011 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5012 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005013 EXPECT_THAT(
5014 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005015 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005016
5017 // Request next frame be a key-frame.
5018 video_stream_encoder_->SendKeyFrame();
5019 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5020 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005021 EXPECT_THAT(
5022 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005023 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005024
5025 video_stream_encoder_->Stop();
5026}
5027
5028TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5029 // Setup simulcast with three streams.
5030 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01005031 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005032 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5033 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5034 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005035 // Wait for all three layers before triggering event.
5036 sink_.SetNumExpectedLayers(3);
5037
5038 // First frame is always keyframe.
5039 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5040 WaitForEncodedFrame(1);
5041 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005042 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5043 VideoFrameType::kVideoFrameKey,
5044 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005045
5046 // Insert delta frame.
5047 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5048 WaitForEncodedFrame(2);
5049 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005050 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5051 VideoFrameType::kVideoFrameDelta,
5052 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005053
5054 // Request next frame be a key-frame.
5055 // Only first stream is configured to produce key-frame.
5056 video_stream_encoder_->SendKeyFrame();
5057 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5058 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005059
5060 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5061 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005062 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005063 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005064 VideoFrameType::kVideoFrameKey,
5065 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005066
5067 video_stream_encoder_->Stop();
5068}
5069
5070TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5071 // Configure internal source factory and setup test again.
5072 encoder_factory_.SetHasInternalSource(true);
5073 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005074 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005075 DataRate::BitsPerSec(kTargetBitrateBps),
5076 DataRate::BitsPerSec(kTargetBitrateBps),
5077 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005078
5079 // Call encoder directly, simulating internal source where encoded frame
5080 // callback in VideoStreamEncoder is called despite no OnFrame().
5081 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5082 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005083 EXPECT_THAT(
5084 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005085 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005086
Niels Möller8f7ce222019-03-21 15:43:58 +01005087 const std::vector<VideoFrameType> kDeltaFrame = {
5088 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005089 // Need to set timestamp manually since manually for injected frame.
5090 VideoFrame frame = CreateFrame(101, nullptr);
5091 frame.set_timestamp(101);
5092 fake_encoder_.InjectFrame(frame, false);
5093 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005094 EXPECT_THAT(
5095 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005096 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005097
5098 // Request key-frame. The forces a dummy frame down into the encoder.
5099 fake_encoder_.ExpectNullFrame();
5100 video_stream_encoder_->SendKeyFrame();
5101 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005102 EXPECT_THAT(
5103 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005104 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005105
5106 video_stream_encoder_->Stop();
5107}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005108
5109TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5110 // Configure internal source factory and setup test again.
5111 encoder_factory_.SetHasInternalSource(true);
5112 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005113 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005114 DataRate::BitsPerSec(kTargetBitrateBps),
5115 DataRate::BitsPerSec(kTargetBitrateBps),
5116 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005117
5118 int64_t timestamp = 1;
5119 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005120 image.SetEncodedData(
5121 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005122 image.capture_time_ms_ = ++timestamp;
5123 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5124 const int64_t kEncodeFinishDelayMs = 10;
5125 image.timing_.encode_start_ms = timestamp;
5126 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5127 fake_encoder_.InjectEncodedImage(image);
5128 // Wait for frame without incrementing clock.
5129 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5130 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5131 // capture timestamp should be kEncodeFinishDelayMs in the past.
5132 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5133 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5134 kEncodeFinishDelayMs);
5135
5136 video_stream_encoder_->Stop();
5137}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005138
5139TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5140 // Configure internal source factory and setup test again.
5141 encoder_factory_.SetHasInternalSource(true);
5142 ResetEncoder("H264", 1, 1, 1, false);
5143
5144 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5145 image._frameType = VideoFrameType::kVideoFrameKey;
5146
5147 CodecSpecificInfo codec_specific_info;
5148 codec_specific_info.codecType = kVideoCodecH264;
5149
5150 RTPFragmentationHeader fragmentation;
5151 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5152 fragmentation.fragmentationOffset[0] = 4;
5153 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5154
5155 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5156 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5157
5158 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5159 testing::ElementsAreArray(optimal_sps));
5160 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5161 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5162 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5163 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5164
5165 video_stream_encoder_->Stop();
5166}
5167
5168TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5169 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5170 0x00, 0x00, 0x03, 0x03, 0xF4,
5171 0x05, 0x03, 0xC7, 0xC0};
5172
5173 // Configure internal source factory and setup test again.
5174 encoder_factory_.SetHasInternalSource(true);
5175 ResetEncoder("H264", 1, 1, 1, false);
5176
5177 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5178 image._frameType = VideoFrameType::kVideoFrameKey;
5179
5180 CodecSpecificInfo codec_specific_info;
5181 codec_specific_info.codecType = kVideoCodecH264;
5182
5183 RTPFragmentationHeader fragmentation;
5184 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5185 fragmentation.fragmentationOffset[0] = 4;
5186 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5187
5188 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5189 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5190
5191 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5192 testing::ElementsAreArray(optimal_sps));
5193 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5194 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5195 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5196 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5197
5198 video_stream_encoder_->Stop();
5199}
5200
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005201TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5202 const int kFrameWidth = 1280;
5203 const int kFrameHeight = 720;
5204 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5205
5206 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005207 DataRate::BitsPerSec(kTargetBitrateBps),
5208 DataRate::BitsPerSec(kTargetBitrateBps),
5209 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005210 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5211
5212 // Insert a first video frame. It should be dropped because of downscale in
5213 // resolution.
5214 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5215 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5216 frame.set_rotation(kVideoRotation_270);
5217 video_source_.IncomingCapturedFrame(frame);
5218
5219 ExpectDroppedFrame();
5220
5221 // Second frame is downscaled.
5222 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5223 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5224 frame.set_rotation(kVideoRotation_90);
5225 video_source_.IncomingCapturedFrame(frame);
5226
5227 WaitForEncodedFrame(timestamp_ms);
5228 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5229
5230 // Insert another frame, also downscaled.
5231 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5232 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5233 frame.set_rotation(kVideoRotation_180);
5234 video_source_.IncomingCapturedFrame(frame);
5235
5236 WaitForEncodedFrame(timestamp_ms);
5237 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5238
5239 video_stream_encoder_->Stop();
5240}
5241
Erik Språng5056af02019-09-02 15:53:11 +02005242TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5243 const int kFrameWidth = 320;
5244 const int kFrameHeight = 180;
5245
5246 // Initial rate.
5247 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005248 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5249 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5250 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005251 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005252 /*rtt_ms=*/0,
5253 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005254
5255 // Insert a first video frame so that encoder gets configured.
5256 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5257 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5258 frame.set_rotation(kVideoRotation_270);
5259 video_source_.IncomingCapturedFrame(frame);
5260 WaitForEncodedFrame(timestamp_ms);
5261
5262 // Set a target rate below the minimum allowed by the codec settings.
5263 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005264 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5265 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Erik Språng5056af02019-09-02 15:53:11 +02005266 video_stream_encoder_->OnBitrateUpdated(
5267 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005268 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005269 /*link_allocation=*/target_rate,
5270 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005271 /*rtt_ms=*/0,
5272 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5274
5275 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5276 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5277 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005278 DataRate allocation_sum =
5279 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005280 EXPECT_EQ(min_rate, allocation_sum);
5281 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5282
5283 video_stream_encoder_->Stop();
5284}
5285
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005286TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005287 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005288 DataRate::BitsPerSec(kTargetBitrateBps),
5289 DataRate::BitsPerSec(kTargetBitrateBps),
5290 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005291 // Capture a frame and wait for it to synchronize with the encoder thread.
5292 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5293 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5294 WaitForEncodedFrame(1);
5295
5296 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5297 ASSERT_TRUE(prev_rate_settings.has_value());
5298 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5299 kDefaultFramerate);
5300
5301 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5302 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5303 timestamp_ms += 1000 / kDefaultFramerate;
5304 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5305 WaitForEncodedFrame(timestamp_ms);
5306 }
5307 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5308 kDefaultFramerate);
5309 // Capture larger frame to trigger a reconfigure.
5310 codec_height_ *= 2;
5311 codec_width_ *= 2;
5312 timestamp_ms += 1000 / kDefaultFramerate;
5313 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5314 WaitForEncodedFrame(timestamp_ms);
5315
5316 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5317 auto current_rate_settings =
5318 fake_encoder_.GetAndResetLastRateControlSettings();
5319 // Ensure we have actually reconfigured twice
5320 // The rate settings should have been set again even though
5321 // they haven't changed.
5322 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005323 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005324
5325 video_stream_encoder_->Stop();
5326}
5327
philipeld9cc8c02019-09-16 14:53:40 +02005328struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5329 MOCK_METHOD0(RequestEncoderFallback, void());
5330 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 11:30:00 +01005331 MOCK_METHOD1(RequestEncoderSwitch,
5332 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 14:53:40 +02005333};
5334
5335TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5336 constexpr int kDontCare = 100;
5337
5338 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5339 video_send_config_.encoder_settings.encoder_switch_request_callback =
5340 &switch_callback;
5341 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5342 encoder_config.codec_type = kVideoCodecVP8;
5343 webrtc::test::ScopedFieldTrials field_trial(
5344 "WebRTC-NetworkCondition-EncoderSwitch/"
5345 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5346 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5347
5348 // Reset encoder for new configuration to take effect.
5349 ConfigureEncoder(std::move(encoder_config));
5350
5351 // Send one frame to trigger ReconfigureEncoder.
5352 video_source_.IncomingCapturedFrame(
5353 CreateFrame(kDontCare, kDontCare, kDontCare));
5354
5355 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005356 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5357 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005358 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005359 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005360
5361 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005362 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5363 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5364 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005365 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005366 /*rtt_ms=*/0,
5367 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005368
5369 video_stream_encoder_->Stop();
5370}
5371
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005372TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5373 constexpr int kDontCare = 100;
5374
5375 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5376 video_send_config_.encoder_settings.encoder_switch_request_callback =
5377 &switch_callback;
5378 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5379 encoder_config.codec_type = kVideoCodecVP8;
5380 webrtc::test::ScopedFieldTrials field_trial(
5381 "WebRTC-NetworkCondition-EncoderSwitch/"
5382 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5383 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5384
5385 // Reset encoder for new configuration to take effect.
5386 ConfigureEncoder(std::move(encoder_config));
5387
5388 // Send one frame to trigger ReconfigureEncoder.
5389 video_source_.IncomingCapturedFrame(
5390 CreateFrame(kDontCare, kDontCare, kDontCare));
5391
5392 using Config = EncoderSwitchRequestCallback::Config;
5393 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5394 .Times(0);
5395
5396 video_stream_encoder_->OnBitrateUpdated(
5397 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5398 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5399 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5400 /*fraction_lost=*/0,
5401 /*rtt_ms=*/0,
5402 /*cwnd_reduce_ratio=*/0);
5403
5404 video_stream_encoder_->Stop();
5405}
5406
philipeld9cc8c02019-09-16 14:53:40 +02005407TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5408 constexpr int kSufficientBitrateToNotDrop = 1000;
5409 constexpr int kHighRes = 500;
5410 constexpr int kLowRes = 100;
5411
5412 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5413 video_send_config_.encoder_settings.encoder_switch_request_callback =
5414 &switch_callback;
5415 webrtc::test::ScopedFieldTrials field_trial(
5416 "WebRTC-NetworkCondition-EncoderSwitch/"
5417 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5418 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5419 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5420 encoder_config.codec_type = kVideoCodecH264;
5421
5422 // Reset encoder for new configuration to take effect.
5423 ConfigureEncoder(std::move(encoder_config));
5424
5425 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5426 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5427 // not fail.
5428 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005429 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5430 /*stable_target_bitrate=*/
5431 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5432 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005433 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005434 /*rtt_ms=*/0,
5435 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005436
5437 // Send one frame to trigger ReconfigureEncoder.
5438 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5439 WaitForEncodedFrame(1);
5440
5441 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005442 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5443 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005444 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005445 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005446
5447 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5448 WaitForEncodedFrame(2);
5449
5450 video_stream_encoder_->Stop();
5451}
5452
philipel9b058032020-02-10 11:30:00 +01005453TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5454 constexpr int kDontCare = 100;
5455 StrictMock<MockEncoderSelector> encoder_selector;
5456 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5457 &fake_encoder_, &encoder_selector);
5458 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5459
5460 // Reset encoder for new configuration to take effect.
5461 ConfigureEncoder(video_encoder_config_.Copy());
5462
5463 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5464
5465 video_source_.IncomingCapturedFrame(
5466 CreateFrame(kDontCare, kDontCare, kDontCare));
5467 video_stream_encoder_->Stop();
5468
5469 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5470 // to it's factory, so in order for the encoder instance in the
5471 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5472 // reset the |video_stream_encoder_| here.
5473 video_stream_encoder_.reset();
5474}
5475
5476TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5477 constexpr int kDontCare = 100;
5478
5479 NiceMock<MockEncoderSelector> encoder_selector;
5480 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5481 video_send_config_.encoder_settings.encoder_switch_request_callback =
5482 &switch_callback;
5483 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5484 &fake_encoder_, &encoder_selector);
5485 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5486
5487 // Reset encoder for new configuration to take effect.
5488 ConfigureEncoder(video_encoder_config_.Copy());
5489
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005490 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005491 .WillByDefault(Return(SdpVideoFormat("AV1")));
5492 EXPECT_CALL(switch_callback,
5493 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5494 Field(&SdpVideoFormat::name, "AV1"))));
5495
5496 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005497 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5498 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5499 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005500 /*fraction_lost=*/0,
5501 /*rtt_ms=*/0,
5502 /*cwnd_reduce_ratio=*/0);
5503
5504 video_stream_encoder_->Stop();
5505}
5506
5507TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5508 constexpr int kSufficientBitrateToNotDrop = 1000;
5509 constexpr int kDontCare = 100;
5510
5511 NiceMock<MockVideoEncoder> video_encoder;
5512 NiceMock<MockEncoderSelector> encoder_selector;
5513 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5514 video_send_config_.encoder_settings.encoder_switch_request_callback =
5515 &switch_callback;
5516 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5517 &video_encoder, &encoder_selector);
5518 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5519
5520 // Reset encoder for new configuration to take effect.
5521 ConfigureEncoder(video_encoder_config_.Copy());
5522
5523 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5524 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5525 // not fail.
5526 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005527 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5528 /*stable_target_bitrate=*/
5529 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5530 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005531 /*fraction_lost=*/0,
5532 /*rtt_ms=*/0,
5533 /*cwnd_reduce_ratio=*/0);
5534
5535 ON_CALL(video_encoder, Encode(_, _))
5536 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5537 ON_CALL(encoder_selector, OnEncoderBroken())
5538 .WillByDefault(Return(SdpVideoFormat("AV2")));
5539
5540 rtc::Event encode_attempted;
5541 EXPECT_CALL(switch_callback,
5542 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5543 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5544 EXPECT_EQ(format.name, "AV2");
5545 encode_attempted.Set();
5546 });
5547
5548 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5549 encode_attempted.Wait(3000);
5550
5551 video_stream_encoder_->Stop();
5552
5553 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5554 // to it's factory, so in order for the encoder instance in the
5555 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5556 // reset the |video_stream_encoder_| here.
5557 video_stream_encoder_.reset();
5558}
5559
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005560TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005561 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005562 const int kFrameWidth = 320;
5563 const int kFrameHeight = 180;
5564
5565 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005566 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005567 video_stream_encoder_->OnBitrateUpdated(
5568 /*target_bitrate=*/rate,
5569 /*stable_target_bitrate=*/rate,
5570 /*link_allocation=*/rate,
5571 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005572 /*rtt_ms=*/0,
5573 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005574
5575 // Insert a first video frame so that encoder gets configured.
5576 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5577 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5578 frame.set_rotation(kVideoRotation_270);
5579 video_source_.IncomingCapturedFrame(frame);
5580 WaitForEncodedFrame(timestamp_ms);
5581 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5582
5583 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005584 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005585 video_stream_encoder_->OnBitrateUpdated(
5586 /*target_bitrate=*/new_stable_rate,
5587 /*stable_target_bitrate=*/new_stable_rate,
5588 /*link_allocation=*/rate,
5589 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005590 /*rtt_ms=*/0,
5591 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005592 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5593 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5594 video_stream_encoder_->Stop();
5595}
5596
5597TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005598 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005599 const int kFrameWidth = 320;
5600 const int kFrameHeight = 180;
5601
5602 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005603 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005604 video_stream_encoder_->OnBitrateUpdated(
5605 /*target_bitrate=*/rate,
5606 /*stable_target_bitrate=*/rate,
5607 /*link_allocation=*/rate,
5608 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005609 /*rtt_ms=*/0,
5610 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005611
5612 // Insert a first video frame so that encoder gets configured.
5613 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5614 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5615 frame.set_rotation(kVideoRotation_270);
5616 video_source_.IncomingCapturedFrame(frame);
5617 WaitForEncodedFrame(timestamp_ms);
5618 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5619
5620 // Set a higher target rate without changing the link_allocation. Should not
5621 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005622 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005623 video_stream_encoder_->OnBitrateUpdated(
5624 /*target_bitrate=*/rate,
5625 /*stable_target_bitrate=*/new_stable_rate,
5626 /*link_allocation=*/rate,
5627 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005628 /*rtt_ms=*/0,
5629 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005630 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5631 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5632 video_stream_encoder_->Stop();
5633}
5634
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005635TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5636 test::ScopedFieldTrials field_trials(
5637 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5638 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5639 const int kFramerateFps = 30;
5640 const int kWidth = 1920;
5641 const int kHeight = 1080;
5642 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5643 // Works on screenshare mode.
5644 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5645 // We rely on the automatic resolution adaptation, but we handle framerate
5646 // adaptation manually by mocking the stats proxy.
5647 video_source_.set_adaptation_enabled(true);
5648
5649 // BALANCED degradation preference is required for this feature.
5650 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005651 DataRate::BitsPerSec(kTargetBitrateBps),
5652 DataRate::BitsPerSec(kTargetBitrateBps),
5653 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005654 video_stream_encoder_->SetSource(&video_source_,
5655 webrtc::DegradationPreference::BALANCED);
5656 VerifyNoLimitation(video_source_.sink_wants());
5657
5658 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5659 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5660
5661 // Pass enough frames with the full update to trigger animation detection.
5662 for (int i = 0; i < kNumFrames; ++i) {
5663 int64_t timestamp_ms =
5664 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5665 frame.set_ntp_time_ms(timestamp_ms);
5666 frame.set_timestamp_us(timestamp_ms * 1000);
5667 video_source_.IncomingCapturedFrame(frame);
5668 WaitForEncodedFrame(timestamp_ms);
5669 }
5670
5671 // Resolution should be limited.
5672 rtc::VideoSinkWants expected;
5673 expected.max_framerate_fps = kFramerateFps;
5674 expected.max_pixel_count = 1280 * 720 + 1;
5675 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5676
5677 // Pass one frame with no known update.
5678 // Resolution cap should be removed immediately.
5679 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5680 frame.set_ntp_time_ms(timestamp_ms);
5681 frame.set_timestamp_us(timestamp_ms * 1000);
5682 frame.clear_update_rect();
5683
5684 video_source_.IncomingCapturedFrame(frame);
5685 WaitForEncodedFrame(timestamp_ms);
5686
5687 // Resolution should be unlimited now.
5688 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5689
5690 video_stream_encoder_->Stop();
5691}
5692
perkj26091b12016-09-01 01:17:40 -07005693} // namespace webrtc