blob: 1cf943689271bee698a7a17098b3cce9c7f6cd49 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080020#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010022#include "api/video/video_bitrate_allocation.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020023#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010024#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020025#include "common_video/h264/h264_common.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020027#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010029#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080030#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010033#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020034#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "system_wrappers/include/sleep.h"
36#include "test/encoder_settings.h"
37#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020038#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "test/frame_generator.h"
40#include "test/gmock.h"
41#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020042#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020043#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070044
45namespace webrtc {
46
sprangb1ca0732017-02-01 08:38:12 -080047using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080048using ::testing::_;
kthelgason876222f2016-11-29 01:44:11 -080049
perkj803d97f2016-11-01 11:45:46 -070050namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020051const int kMinPixelsPerFrame = 320 * 180;
52const int kMinFramerateFps = 2;
53const int kMinBalancedFramerateFps = 7;
54const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080055const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010056const uint32_t kTargetBitrateBps = 1000000;
57const uint32_t kSimulcastTargetBitrateBps = 3150000;
58const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080059const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070060const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020061const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080062
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020063uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
64 0x00, 0x00, 0x03, 0x03, 0xF4,
65 0x05, 0x03, 0xC7, 0xE0, 0x1B,
66 0x41, 0x10, 0x8D, 0x00};
67
perkj803d97f2016-11-01 11:45:46 -070068class TestBuffer : public webrtc::I420Buffer {
69 public:
70 TestBuffer(rtc::Event* event, int width, int height)
71 : I420Buffer(width, height), event_(event) {}
72
73 private:
74 friend class rtc::RefCountedObject<TestBuffer>;
75 ~TestBuffer() override {
76 if (event_)
77 event_->Set();
78 }
79 rtc::Event* const event_;
80};
81
Niels Möller7dc26b72017-12-06 10:27:48 +010082class CpuOveruseDetectorProxy : public OveruseFrameDetector {
83 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020084 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
85 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010086 last_target_framerate_fps_(-1) {}
87 virtual ~CpuOveruseDetectorProxy() {}
88
89 void OnTargetFramerateUpdated(int framerate_fps) override {
90 rtc::CritScope cs(&lock_);
91 last_target_framerate_fps_ = framerate_fps;
92 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
93 }
94
95 int GetLastTargetFramerate() {
96 rtc::CritScope cs(&lock_);
97 return last_target_framerate_fps_;
98 }
99
Niels Möller4db138e2018-04-19 09:04:13 +0200100 CpuOveruseOptions GetOptions() { return options_; }
101
Niels Möller7dc26b72017-12-06 10:27:48 +0100102 private:
103 rtc::CriticalSection lock_;
104 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
105};
106
mflodmancc3d4422017-08-03 08:27:51 -0700107class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700108 public:
Niels Möller213618e2018-07-24 09:29:58 +0200109 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200110 const VideoStreamEncoderSettings& settings,
111 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100112 : VideoStreamEncoder(Clock::GetRealTimeClock(),
113 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200114 stats_proxy,
115 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200116 std::unique_ptr<OveruseFrameDetector>(
117 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100118 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200119 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700120
sprangb1ca0732017-02-01 08:38:12 -0800121 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100122 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800123 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800124 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700125 event.Set();
126 });
perkj070ba852017-02-16 15:46:27 -0800127 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700128 }
129
kthelgason2fc52542017-03-03 00:24:41 -0800130 // This is used as a synchronisation mechanism, to make sure that the
131 // encoder queue is not blocked before we start sending it frames.
132 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100133 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200134 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800135 ASSERT_TRUE(event.Wait(5000));
136 }
137
sprangb1ca0732017-02-01 08:38:12 -0800138 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800139
sprangb1ca0732017-02-01 08:38:12 -0800140 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800141
sprangb1ca0732017-02-01 08:38:12 -0800142 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800143
sprangb1ca0732017-02-01 08:38:12 -0800144 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700145
Niels Möller7dc26b72017-12-06 10:27:48 +0100146 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700147};
148
asapersson5f7226f2016-11-25 04:37:00 -0800149class VideoStreamFactory
150 : public VideoEncoderConfig::VideoStreamFactoryInterface {
151 public:
sprangfda496a2017-06-15 04:21:07 -0700152 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
153 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800154 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700155 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800156 }
157
158 private:
159 std::vector<VideoStream> CreateEncoderStreams(
160 int width,
161 int height,
162 const VideoEncoderConfig& encoder_config) override {
163 std::vector<VideoStream> streams =
164 test::CreateVideoStreams(width, height, encoder_config);
165 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100166 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700167 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800168 }
169 return streams;
170 }
sprangfda496a2017-06-15 04:21:07 -0700171
asapersson5f7226f2016-11-25 04:37:00 -0800172 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700173 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800174};
175
sprangb1ca0732017-02-01 08:38:12 -0800176class AdaptingFrameForwarder : public test::FrameForwarder {
177 public:
178 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700179 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800180
181 void set_adaptation_enabled(bool enabled) {
182 rtc::CritScope cs(&crit_);
183 adaptation_enabled_ = enabled;
184 }
185
asaperssonfab67072017-04-04 05:51:49 -0700186 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800187 rtc::CritScope cs(&crit_);
188 return adaptation_enabled_;
189 }
190
asapersson09f05612017-05-15 23:40:18 -0700191 rtc::VideoSinkWants last_wants() const {
192 rtc::CritScope cs(&crit_);
193 return last_wants_;
194 }
195
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200196 absl::optional<int> last_sent_width() const { return last_width_; }
197 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800198
sprangb1ca0732017-02-01 08:38:12 -0800199 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
200 int cropped_width = 0;
201 int cropped_height = 0;
202 int out_width = 0;
203 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700204 if (adaption_enabled()) {
205 if (adapter_.AdaptFrameResolution(
206 video_frame.width(), video_frame.height(),
207 video_frame.timestamp_us() * 1000, &cropped_width,
208 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100209 VideoFrame adapted_frame =
210 VideoFrame::Builder()
211 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
212 nullptr, out_width, out_height))
213 .set_timestamp_rtp(99)
214 .set_timestamp_ms(99)
215 .set_rotation(kVideoRotation_0)
216 .build();
sprangc5d62e22017-04-02 23:53:04 -0700217 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
218 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800219 last_width_.emplace(adapted_frame.width());
220 last_height_.emplace(adapted_frame.height());
221 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200222 last_width_ = absl::nullopt;
223 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700224 }
sprangb1ca0732017-02-01 08:38:12 -0800225 } else {
226 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800227 last_width_.emplace(video_frame.width());
228 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800229 }
230 }
231
232 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
233 const rtc::VideoSinkWants& wants) override {
234 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700235 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700236 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
237 wants.max_pixel_count,
238 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800239 test::FrameForwarder::AddOrUpdateSink(sink, wants);
240 }
sprangb1ca0732017-02-01 08:38:12 -0800241 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700242 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
243 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200244 absl::optional<int> last_width_;
245 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800246};
sprangc5d62e22017-04-02 23:53:04 -0700247
Niels Möller213618e2018-07-24 09:29:58 +0200248// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700249class MockableSendStatisticsProxy : public SendStatisticsProxy {
250 public:
251 MockableSendStatisticsProxy(Clock* clock,
252 const VideoSendStream::Config& config,
253 VideoEncoderConfig::ContentType content_type)
254 : SendStatisticsProxy(clock, config, content_type) {}
255
256 VideoSendStream::Stats GetStats() override {
257 rtc::CritScope cs(&lock_);
258 if (mock_stats_)
259 return *mock_stats_;
260 return SendStatisticsProxy::GetStats();
261 }
262
Niels Möller213618e2018-07-24 09:29:58 +0200263 int GetInputFrameRate() const override {
264 rtc::CritScope cs(&lock_);
265 if (mock_stats_)
266 return mock_stats_->input_frame_rate;
267 return SendStatisticsProxy::GetInputFrameRate();
268 }
sprangc5d62e22017-04-02 23:53:04 -0700269 void SetMockStats(const VideoSendStream::Stats& stats) {
270 rtc::CritScope cs(&lock_);
271 mock_stats_.emplace(stats);
272 }
273
274 void ResetMockStats() {
275 rtc::CritScope cs(&lock_);
276 mock_stats_.reset();
277 }
278
279 private:
280 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200281 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700282};
283
sprang4847ae62017-06-27 07:06:52 -0700284class MockBitrateObserver : public VideoBitrateAllocationObserver {
285 public:
Erik Språng566124a2018-04-23 12:32:22 +0200286 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700287};
288
perkj803d97f2016-11-01 11:45:46 -0700289} // namespace
290
mflodmancc3d4422017-08-03 08:27:51 -0700291class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700292 public:
293 static const int kDefaultTimeoutMs = 30 * 1000;
294
mflodmancc3d4422017-08-03 08:27:51 -0700295 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700296 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700297 codec_width_(320),
298 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200299 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200300 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700301 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200302 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800303 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700304 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700305 Clock::GetRealTimeClock(),
306 video_send_config_,
307 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700308 sink_(&fake_encoder_) {}
309
310 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700311 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700312 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200313 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800314 video_send_config_.encoder_settings.bitrate_allocator_factory =
315 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200316 video_send_config_.rtp.payload_name = "FAKE";
317 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700318
Per512ecb32016-09-23 15:52:06 +0200319 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200320 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700321 video_encoder_config.video_stream_factory =
322 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100323 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700324
325 // Framerate limit is specified by the VideoStreamFactory.
326 std::vector<VideoStream> streams =
327 video_encoder_config.video_stream_factory->CreateEncoderStreams(
328 codec_width_, codec_height_, video_encoder_config);
329 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200330 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700331
Niels Möllerf1338562018-04-26 09:51:47 +0200332 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800333 }
334
Niels Möllerf1338562018-04-26 09:51:47 +0200335 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700336 if (video_stream_encoder_)
337 video_stream_encoder_->Stop();
338 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200339 stats_proxy_.get(), video_send_config_.encoder_settings,
340 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700341 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
342 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700343 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700344 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
345 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200346 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700347 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800348 }
349
350 void ResetEncoder(const std::string& payload_name,
351 size_t num_streams,
352 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700353 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700354 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200355 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800356
357 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200358 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800359 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100360 video_encoder_config.max_bitrate_bps =
361 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800362 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700363 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
364 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700365 video_encoder_config.content_type =
366 screenshare ? VideoEncoderConfig::ContentType::kScreen
367 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700368 if (payload_name == "VP9") {
369 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
370 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
371 video_encoder_config.encoder_specific_settings =
372 new rtc::RefCountedObject<
373 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
374 }
Niels Möllerf1338562018-04-26 09:51:47 +0200375 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700376 }
377
sprang57c2fff2017-01-16 06:24:02 -0800378 VideoFrame CreateFrame(int64_t ntp_time_ms,
379 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100380 VideoFrame frame =
381 VideoFrame::Builder()
382 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
383 destruction_event, codec_width_, codec_height_))
384 .set_timestamp_rtp(99)
385 .set_timestamp_ms(99)
386 .set_rotation(kVideoRotation_0)
387 .build();
sprang57c2fff2017-01-16 06:24:02 -0800388 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700389 return frame;
390 }
391
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100392 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
393 rtc::Event* destruction_event,
394 int offset_x) const {
395 VideoFrame frame =
396 VideoFrame::Builder()
397 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
398 destruction_event, codec_width_, codec_height_))
399 .set_timestamp_rtp(99)
400 .set_timestamp_ms(99)
401 .set_rotation(kVideoRotation_0)
402 .set_update_rect({offset_x, 0, 1, 1})
403 .build();
404 frame.set_ntp_time_ms(ntp_time_ms);
405 return frame;
406 }
407
sprang57c2fff2017-01-16 06:24:02 -0800408 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100409 VideoFrame frame =
410 VideoFrame::Builder()
411 .set_video_frame_buffer(
412 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
413 .set_timestamp_rtp(99)
414 .set_timestamp_ms(99)
415 .set_rotation(kVideoRotation_0)
416 .build();
sprang57c2fff2017-01-16 06:24:02 -0800417 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700418 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700419 return frame;
420 }
421
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100422 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
423 MockBitrateObserver bitrate_observer;
424 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
425
426 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
427 .Times(1);
428 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +0200429 DataRate::bps(kTargetBitrateBps), 0,
430 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100431
432 video_source_.IncomingCapturedFrame(
433 CreateFrame(1, codec_width_, codec_height_));
434 WaitForEncodedFrame(1);
435 }
436
asapersson02465b82017-04-10 01:12:52 -0700437 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700438 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700439 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
440 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700441 }
442
asapersson09f05612017-05-15 23:40:18 -0700443 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
444 const rtc::VideoSinkWants& wants2) {
445 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
446 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
447 }
448
Åsa Persson8c1bf952018-09-13 10:42:19 +0200449 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
450 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
451 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
452 EXPECT_FALSE(wants.target_pixel_count);
453 }
454
asapersson09f05612017-05-15 23:40:18 -0700455 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
456 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200457 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700458 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
459 EXPECT_GT(wants1.max_pixel_count, 0);
460 }
461
462 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
463 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200464 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700465 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
466 }
467
asaperssonf7e294d2017-06-13 23:25:22 -0700468 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
469 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200470 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700471 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
472 }
473
474 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
475 const rtc::VideoSinkWants& wants2) {
476 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
477 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
478 }
479
480 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
481 const rtc::VideoSinkWants& wants2) {
482 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
483 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
484 }
485
486 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
487 const rtc::VideoSinkWants& wants2) {
488 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
489 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
490 EXPECT_GT(wants1.max_pixel_count, 0);
491 }
492
493 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
494 const rtc::VideoSinkWants& wants2) {
495 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
496 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
497 }
498
asapersson09f05612017-05-15 23:40:18 -0700499 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
500 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200501 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700502 EXPECT_LT(wants.max_pixel_count, pixel_count);
503 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700504 }
505
506 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
507 EXPECT_LT(wants.max_framerate_fps, fps);
508 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
509 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700510 }
511
asaperssonf7e294d2017-06-13 23:25:22 -0700512 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
513 int expected_fps) {
514 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
515 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
516 EXPECT_FALSE(wants.target_pixel_count);
517 }
518
Jonathan Yubc771b72017-12-08 17:04:29 -0800519 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
520 int last_frame_pixels) {
521 // Balanced mode should always scale FPS to the desired range before
522 // attempting to scale resolution.
523 int fps_limit = wants.max_framerate_fps;
524 if (last_frame_pixels <= 320 * 240) {
525 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
526 } else if (last_frame_pixels <= 480 * 270) {
527 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
528 } else if (last_frame_pixels <= 640 * 480) {
529 EXPECT_LE(15, fps_limit);
530 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200531 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800532 }
533 }
534
sprang4847ae62017-06-27 07:06:52 -0700535 void WaitForEncodedFrame(int64_t expected_ntp_time) {
536 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200537 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700538 }
539
540 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
541 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200542 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700543 return ok;
544 }
545
546 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
547 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200548 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700549 }
550
551 void ExpectDroppedFrame() {
552 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200553 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700554 }
555
556 bool WaitForFrame(int64_t timeout_ms) {
557 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200558 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700559 return ok;
560 }
561
perkj26091b12016-09-01 01:17:40 -0700562 class TestEncoder : public test::FakeEncoder {
563 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100564 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700565
asaperssonfab67072017-04-04 05:51:49 -0700566 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800567 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700568 return config_;
569 }
570
571 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800572 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700573 block_next_encode_ = true;
574 }
575
Erik Språngaed30702018-11-05 12:57:17 +0100576 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800577 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100578 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100579 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100580 if (quality_scaling_) {
581 info.scaling_settings =
582 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
583 }
584 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100585 for (int i = 0; i < kMaxSpatialLayers; ++i) {
586 if (temporal_layers_supported_[i]) {
587 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
588 info.fps_allocation[i].resize(num_layers);
589 }
590 }
Erik Språngaed30702018-11-05 12:57:17 +0100591 }
592 return info;
kthelgason876222f2016-11-29 01:44:11 -0800593 }
594
Erik Språngb7cb7b52019-02-26 15:52:33 +0100595 int32_t RegisterEncodeCompleteCallback(
596 EncodedImageCallback* callback) override {
597 rtc::CritScope lock(&local_crit_sect_);
598 encoded_image_callback_ = callback;
599 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
600 }
601
perkjfa10b552016-10-02 23:45:26 -0700602 void ContinueEncode() { continue_encode_event_.Set(); }
603
604 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
605 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800606 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700607 EXPECT_EQ(timestamp_, timestamp);
608 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
609 }
610
kthelgason2fc52542017-03-03 00:24:41 -0800611 void SetQualityScaling(bool b) {
612 rtc::CritScope lock(&local_crit_sect_);
613 quality_scaling_ = b;
614 }
kthelgasonad9010c2017-02-14 00:46:51 -0800615
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100616 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
617 rtc::CritScope lock(&local_crit_sect_);
618 is_hardware_accelerated_ = is_hardware_accelerated;
619 }
620
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100621 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
622 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
623 rtc::CritScope lock(&local_crit_sect_);
624 temporal_layers_supported_[spatial_idx] = supported;
625 }
626
sprangfe627f32017-03-29 08:24:59 -0700627 void ForceInitEncodeFailure(bool force_failure) {
628 rtc::CritScope lock(&local_crit_sect_);
629 force_init_encode_failed_ = force_failure;
630 }
631
Niels Möller6bb5ab92019-01-11 11:11:10 +0100632 void SimulateOvershoot(double rate_factor) {
633 rtc::CritScope lock(&local_crit_sect_);
634 rate_factor_ = rate_factor;
635 }
636
Erik Språngd7329ca2019-02-21 21:19:53 +0100637 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100638 rtc::CritScope lock(&local_crit_sect_);
639 return last_framerate_;
640 }
641
Erik Språngd7329ca2019-02-21 21:19:53 +0100642 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100643 rtc::CritScope lock(&local_crit_sect_);
644 return last_update_rect_;
645 }
646
Niels Möller87e2d782019-03-07 10:18:23 +0100647 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100648 rtc::CritScope lock(&local_crit_sect_);
649 return last_frame_types_;
650 }
651
652 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100653 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100654 keyframe ? VideoFrameType::kVideoFrameKey
655 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100656 {
657 rtc::CritScope lock(&local_crit_sect_);
658 last_frame_types_ = frame_type;
659 }
Niels Möllerb859b322019-03-07 12:40:01 +0100660 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100661 }
662
Erik Språngb7cb7b52019-02-26 15:52:33 +0100663 void InjectEncodedImage(const EncodedImage& image) {
664 rtc::CritScope lock(&local_crit_sect_);
665 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
666 }
667
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200668 void InjectEncodedImage(const EncodedImage& image,
669 const CodecSpecificInfo* codec_specific_info,
670 const RTPFragmentationHeader* fragmentation) {
671 rtc::CritScope lock(&local_crit_sect_);
672 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
673 fragmentation);
674 }
675
Erik Språngd7329ca2019-02-21 21:19:53 +0100676 void ExpectNullFrame() {
677 rtc::CritScope lock(&local_crit_sect_);
678 expect_null_frame_ = true;
679 }
680
681 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
682 auto allocation = last_bitrate_allocation_;
683 last_bitrate_allocation_.reset();
684 return allocation;
685 }
686
perkjfa10b552016-10-02 23:45:26 -0700687 private:
perkj26091b12016-09-01 01:17:40 -0700688 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100689 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700690 bool block_encode;
691 {
brandtre78d2662017-01-16 05:57:16 -0800692 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100693 if (expect_null_frame_) {
694 EXPECT_EQ(input_image.timestamp(), 0u);
695 EXPECT_EQ(input_image.width(), 1);
696 last_frame_types_ = *frame_types;
697 expect_null_frame_ = false;
698 } else {
699 EXPECT_GT(input_image.timestamp(), timestamp_);
700 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
701 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
702 }
perkj26091b12016-09-01 01:17:40 -0700703
704 timestamp_ = input_image.timestamp();
705 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700706 last_input_width_ = input_image.width();
707 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700708 block_encode = block_next_encode_;
709 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100710 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100711 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700712 }
Niels Möllerb859b322019-03-07 12:40:01 +0100713 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700714 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700715 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700716 return result;
717 }
718
sprangfe627f32017-03-29 08:24:59 -0700719 int32_t InitEncode(const VideoCodec* config,
720 int32_t number_of_cores,
721 size_t max_payload_size) override {
722 int res =
723 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
724 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100725 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100726 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700727 // Simulate setting up temporal layers, in order to validate the life
728 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100729 Vp8TemporalLayersFactory factory;
730 frame_buffer_controller_ = factory.Create(*config);
sprangfe627f32017-03-29 08:24:59 -0700731 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100732 if (force_init_encode_failed_) {
733 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700734 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100735 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100736
Erik Språngb7cb7b52019-02-26 15:52:33 +0100737 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700738 return res;
739 }
740
Erik Språngb7cb7b52019-02-26 15:52:33 +0100741 int32_t Release() override {
742 rtc::CritScope lock(&local_crit_sect_);
743 EXPECT_NE(initialized_, EncoderState::kUninitialized);
744 initialized_ = EncoderState::kUninitialized;
745 return FakeEncoder::Release();
746 }
747
Erik Språng16cb8f52019-04-12 13:59:09 +0200748 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100749 rtc::CritScope lock(&local_crit_sect_);
750 VideoBitrateAllocation adjusted_rate_allocation;
751 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
752 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200753 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100754 adjusted_rate_allocation.SetBitrate(
755 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200756 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100757 rate_factor_));
758 }
759 }
760 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200761 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
762 last_bitrate_allocation_ = parameters.bitrate;
763 RateControlParameters adjusted_paramters = parameters;
764 adjusted_paramters.bitrate = adjusted_rate_allocation;
765 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100766 }
767
brandtre78d2662017-01-16 05:57:16 -0800768 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100769 enum class EncoderState {
770 kUninitialized,
771 kInitializationFailed,
772 kInitialized
773 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
774 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700775 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700776 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700777 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
778 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
779 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
780 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
781 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100782 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100783 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700784 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100785 absl::optional<bool>
786 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
787 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700788 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100789 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
790 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100791 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100792 VideoFrame::UpdateRect last_update_rect_
793 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100794 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100795 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100796 EncodedImageCallback* encoded_image_callback_
797 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
perkj26091b12016-09-01 01:17:40 -0700798 };
799
mflodmancc3d4422017-08-03 08:27:51 -0700800 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700801 public:
802 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100803 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700804
perkj26091b12016-09-01 01:17:40 -0700805 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700806 EXPECT_TRUE(
807 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
808 }
809
810 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
811 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700812 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700813 if (!encoded_frame_event_.Wait(timeout_ms))
814 return false;
perkj26091b12016-09-01 01:17:40 -0700815 {
816 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800817 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700818 }
819 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700820 return true;
perkj26091b12016-09-01 01:17:40 -0700821 }
822
sprangb1ca0732017-02-01 08:38:12 -0800823 void WaitForEncodedFrame(uint32_t expected_width,
824 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700825 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100826 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700827 }
828
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100829 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700830 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800831 uint32_t width = 0;
832 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800833 {
834 rtc::CritScope lock(&crit_);
835 width = last_width_;
836 height = last_height_;
837 }
838 EXPECT_EQ(expected_height, height);
839 EXPECT_EQ(expected_width, width);
840 }
841
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200842 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
843 VideoRotation rotation;
844 {
845 rtc::CritScope lock(&crit_);
846 rotation = last_rotation_;
847 }
848 EXPECT_EQ(expected_rotation, rotation);
849 }
850
kthelgason2fc52542017-03-03 00:24:41 -0800851 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800852
sprangc5d62e22017-04-02 23:53:04 -0700853 bool WaitForFrame(int64_t timeout_ms) {
854 return encoded_frame_event_.Wait(timeout_ms);
855 }
856
perkj26091b12016-09-01 01:17:40 -0700857 void SetExpectNoFrames() {
858 rtc::CritScope lock(&crit_);
859 expect_frames_ = false;
860 }
861
asaperssonfab67072017-04-04 05:51:49 -0700862 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200863 rtc::CritScope lock(&crit_);
864 return number_of_reconfigurations_;
865 }
866
asaperssonfab67072017-04-04 05:51:49 -0700867 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200868 rtc::CritScope lock(&crit_);
869 return min_transmit_bitrate_bps_;
870 }
871
Erik Språngd7329ca2019-02-21 21:19:53 +0100872 void SetNumExpectedLayers(size_t num_layers) {
873 rtc::CritScope lock(&crit_);
874 num_expected_layers_ = num_layers;
875 }
876
Erik Språngb7cb7b52019-02-26 15:52:33 +0100877 int64_t GetLastCaptureTimeMs() const {
878 rtc::CritScope lock(&crit_);
879 return last_capture_time_ms_;
880 }
881
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200882 std::vector<uint8_t> GetLastEncodedImageData() {
883 rtc::CritScope lock(&crit_);
884 return std::move(last_encoded_image_data_);
885 }
886
887 RTPFragmentationHeader GetLastFragmentation() {
888 rtc::CritScope lock(&crit_);
889 return std::move(last_fragmentation_);
890 }
891
perkj26091b12016-09-01 01:17:40 -0700892 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700893 Result OnEncodedImage(
894 const EncodedImage& encoded_image,
895 const CodecSpecificInfo* codec_specific_info,
896 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200897 rtc::CritScope lock(&crit_);
898 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200899 last_encoded_image_data_ = std::vector<uint8_t>(
900 encoded_image.data(), encoded_image.data() + encoded_image.size());
901 if (fragmentation) {
902 last_fragmentation_.CopyFrom(*fragmentation);
903 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100904 uint32_t timestamp = encoded_image.Timestamp();
905 if (last_timestamp_ != timestamp) {
906 num_received_layers_ = 1;
907 } else {
908 ++num_received_layers_;
909 }
910 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100911 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -0800912 last_width_ = encoded_image._encodedWidth;
913 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200914 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100915 if (num_received_layers_ == num_expected_layers_) {
916 encoded_frame_event_.Set();
917 }
sprangb1ca0732017-02-01 08:38:12 -0800918 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200919 }
920
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100921 void OnEncoderConfigurationChanged(
922 std::vector<VideoStream> streams,
923 VideoEncoderConfig::ContentType content_type,
924 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200925 rtc::CriticalSection crit_;
926 ++number_of_reconfigurations_;
927 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
928 }
929
perkj26091b12016-09-01 01:17:40 -0700930 rtc::CriticalSection crit_;
931 TestEncoder* test_encoder_;
932 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200933 std::vector<uint8_t> last_encoded_image_data_;
934 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -0800935 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100936 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -0800937 uint32_t last_height_ = 0;
938 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200939 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100940 size_t num_expected_layers_ = 1;
941 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700942 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200943 int number_of_reconfigurations_ = 0;
944 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700945 };
946
947 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100948 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200949 int codec_width_;
950 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700951 int max_framerate_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200952 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -0700953 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200954 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800955 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700956 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700957 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800958 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700959 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700960 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700961};
962
mflodmancc3d4422017-08-03 08:27:51 -0700963TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +0200964 video_stream_encoder_->OnBitrateUpdated(
965 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100966 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700967 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700968 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700969 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700970 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700971}
972
mflodmancc3d4422017-08-03 08:27:51 -0700973TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700974 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100975 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200976 // The encoder will cache up to one frame for a short duration. Adding two
977 // frames means that the first frame will be dropped and the second frame will
978 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700979 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200980 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700981 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700982
Erik Språng4c6ca302019-04-08 15:14:01 +0200983 video_stream_encoder_->OnBitrateUpdated(
984 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -0700985
Sebastian Janssona3177052018-04-10 13:05:49 +0200986 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700987 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200988 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
989
990 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700991 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700992}
993
mflodmancc3d4422017-08-03 08:27:51 -0700994TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +0200995 video_stream_encoder_->OnBitrateUpdated(
996 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700997 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700998 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700999
Erik Språng4c6ca302019-04-08 15:14:01 +02001000 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01001001 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001002 // The encoder will cache up to one frame for a short duration. Adding two
1003 // frames means that the first frame will be dropped and the second frame will
1004 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001005 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001006 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001007
Erik Språng4c6ca302019-04-08 15:14:01 +02001008 video_stream_encoder_->OnBitrateUpdated(
1009 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001010 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001011 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1012 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001013 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001014}
1015
mflodmancc3d4422017-08-03 08:27:51 -07001016TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001017 video_stream_encoder_->OnBitrateUpdated(
1018 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001019 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001020 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001021
1022 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001023 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001024
perkja49cbd32016-09-16 07:53:41 -07001025 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001026 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001027 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001028}
1029
mflodmancc3d4422017-08-03 08:27:51 -07001030TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001031 video_stream_encoder_->OnBitrateUpdated(
1032 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001033
perkja49cbd32016-09-16 07:53:41 -07001034 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001035 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001036
mflodmancc3d4422017-08-03 08:27:51 -07001037 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001038 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001039 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001040 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1041 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001042}
1043
mflodmancc3d4422017-08-03 08:27:51 -07001044TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001045 video_stream_encoder_->OnBitrateUpdated(
1046 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001047
1048 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001049 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001050 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001051 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1052 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001053 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1054 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001055 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001056 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001057
mflodmancc3d4422017-08-03 08:27:51 -07001058 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001059}
1060
mflodmancc3d4422017-08-03 08:27:51 -07001061TEST_F(VideoStreamEncoderTest,
1062 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001063 video_stream_encoder_->OnBitrateUpdated(
1064 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001065 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001066
1067 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001068 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001069 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001070 // The encoder will have been configured once when the first frame is
1071 // received.
1072 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001073
1074 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001075 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001076 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001077 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001078 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001079
1080 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001081 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001082 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001083 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001084 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001085
mflodmancc3d4422017-08-03 08:27:51 -07001086 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001087}
1088
mflodmancc3d4422017-08-03 08:27:51 -07001089TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001090 video_stream_encoder_->OnBitrateUpdated(
1091 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001092
1093 // Capture a frame and wait for it to synchronize with the encoder thread.
1094 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001095 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001096 // The encoder will have been configured once.
1097 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001098 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1099 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1100
1101 codec_width_ *= 2;
1102 codec_height_ *= 2;
1103 // Capture a frame with a higher resolution and wait for it to synchronize
1104 // with the encoder thread.
1105 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001106 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001107 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1108 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001109 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001110
mflodmancc3d4422017-08-03 08:27:51 -07001111 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001112}
1113
mflodmancc3d4422017-08-03 08:27:51 -07001114TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001115 EXPECT_TRUE(video_source_.has_sinks());
1116 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001117 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001118 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001119 EXPECT_FALSE(video_source_.has_sinks());
1120 EXPECT_TRUE(new_video_source.has_sinks());
1121
mflodmancc3d4422017-08-03 08:27:51 -07001122 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001123}
1124
mflodmancc3d4422017-08-03 08:27:51 -07001125TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001126 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001127 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001128 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001129 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001130}
1131
Jonathan Yubc771b72017-12-08 17:04:29 -08001132TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1133 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001134 const int kWidth = 1280;
1135 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001136
1137 // We rely on the automatic resolution adaptation, but we handle framerate
1138 // adaptation manually by mocking the stats proxy.
1139 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001140
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001141 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001142 video_stream_encoder_->OnBitrateUpdated(
1143 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001144 video_stream_encoder_->SetSource(&video_source_,
1145 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001146 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001147 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001148 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001149 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1150
Jonathan Yubc771b72017-12-08 17:04:29 -08001151 // Adapt down as far as possible.
1152 rtc::VideoSinkWants last_wants;
1153 int64_t t = 1;
1154 int loop_count = 0;
1155 do {
1156 ++loop_count;
1157 last_wants = video_source_.sink_wants();
1158
1159 // Simulate the framerate we've been asked to adapt to.
1160 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1161 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1162 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1163 mock_stats.input_frame_rate = fps;
1164 stats_proxy_->SetMockStats(mock_stats);
1165
1166 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1167 sink_.WaitForEncodedFrame(t);
1168 t += frame_interval_ms;
1169
mflodmancc3d4422017-08-03 08:27:51 -07001170 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001171 VerifyBalancedModeFpsRange(
1172 video_source_.sink_wants(),
1173 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1174 } while (video_source_.sink_wants().max_pixel_count <
1175 last_wants.max_pixel_count ||
1176 video_source_.sink_wants().max_framerate_fps <
1177 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001178
Jonathan Yubc771b72017-12-08 17:04:29 -08001179 // Verify that we've adapted all the way down.
1180 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001181 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001182 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1183 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001184 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001185 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1186 *video_source_.last_sent_height());
1187 EXPECT_EQ(kMinBalancedFramerateFps,
1188 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001189
Jonathan Yubc771b72017-12-08 17:04:29 -08001190 // Adapt back up the same number of times we adapted down.
1191 for (int i = 0; i < loop_count - 1; ++i) {
1192 last_wants = video_source_.sink_wants();
1193
1194 // Simulate the framerate we've been asked to adapt to.
1195 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1196 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1197 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1198 mock_stats.input_frame_rate = fps;
1199 stats_proxy_->SetMockStats(mock_stats);
1200
1201 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1202 sink_.WaitForEncodedFrame(t);
1203 t += frame_interval_ms;
1204
mflodmancc3d4422017-08-03 08:27:51 -07001205 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001206 VerifyBalancedModeFpsRange(
1207 video_source_.sink_wants(),
1208 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1209 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1210 last_wants.max_pixel_count ||
1211 video_source_.sink_wants().max_framerate_fps >
1212 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001213 }
1214
Åsa Persson8c1bf952018-09-13 10:42:19 +02001215 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001216 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001217 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001218 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1219 EXPECT_EQ((loop_count - 1) * 2,
1220 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001221
mflodmancc3d4422017-08-03 08:27:51 -07001222 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001223}
mflodmancc3d4422017-08-03 08:27:51 -07001224TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001225 video_stream_encoder_->OnBitrateUpdated(
1226 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001227 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001228
sprangc5d62e22017-04-02 23:53:04 -07001229 const int kFrameWidth = 1280;
1230 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001231
Åsa Persson8c1bf952018-09-13 10:42:19 +02001232 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001233
kthelgason5e13d412016-12-01 03:59:51 -08001234 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001235 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001236 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001237 frame_timestamp += kFrameIntervalMs;
1238
perkj803d97f2016-11-01 11:45:46 -07001239 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001240 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001241 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001242 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001243 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001244 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001245
asapersson0944a802017-04-07 00:57:58 -07001246 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001247 // wanted resolution.
1248 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1249 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1250 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001251 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001252
1253 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001254 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001256 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001257
sprangc5d62e22017-04-02 23:53:04 -07001258 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001259 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001260
sprangc5d62e22017-04-02 23:53:04 -07001261 // Force an input frame rate to be available, or the adaptation call won't
1262 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001263 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001264 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001265 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001266 stats_proxy_->SetMockStats(stats);
1267
mflodmancc3d4422017-08-03 08:27:51 -07001268 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001269 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001270 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001271 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001272 frame_timestamp += kFrameIntervalMs;
1273
1274 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001275 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001276 EXPECT_EQ(std::numeric_limits<int>::max(),
1277 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001278 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001279
asapersson02465b82017-04-10 01:12:52 -07001280 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001281 video_stream_encoder_->SetSource(&new_video_source,
1282 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001283 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001284
mflodmancc3d4422017-08-03 08:27:51 -07001285 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001286 new_video_source.IncomingCapturedFrame(
1287 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001288 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001289 frame_timestamp += kFrameIntervalMs;
1290
1291 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001292 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001293
1294 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001295 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001296 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001297 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1298 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001299 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001300 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001301
1302 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001303 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001304 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001305 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1306 EXPECT_EQ(std::numeric_limits<int>::max(),
1307 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001308 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001309
mflodmancc3d4422017-08-03 08:27:51 -07001310 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001311}
1312
mflodmancc3d4422017-08-03 08:27:51 -07001313TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001314 video_stream_encoder_->OnBitrateUpdated(
1315 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001316
asaperssonfab67072017-04-04 05:51:49 -07001317 const int kWidth = 1280;
1318 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001319 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001320 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001321 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1322 EXPECT_FALSE(stats.bw_limited_resolution);
1323 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1324
1325 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001326 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001327 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001328 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001329
1330 stats = stats_proxy_->GetStats();
1331 EXPECT_TRUE(stats.bw_limited_resolution);
1332 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1333
1334 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001335 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001336 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001337 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001338
1339 stats = stats_proxy_->GetStats();
1340 EXPECT_FALSE(stats.bw_limited_resolution);
1341 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1342 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1343
mflodmancc3d4422017-08-03 08:27:51 -07001344 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001345}
1346
mflodmancc3d4422017-08-03 08:27:51 -07001347TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001348 video_stream_encoder_->OnBitrateUpdated(
1349 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001350
1351 const int kWidth = 1280;
1352 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001353 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001354 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001355 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1356 EXPECT_FALSE(stats.cpu_limited_resolution);
1357 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1358
1359 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001360 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001361 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001362 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001363
1364 stats = stats_proxy_->GetStats();
1365 EXPECT_TRUE(stats.cpu_limited_resolution);
1366 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1367
1368 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001369 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001370 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001371 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001372
1373 stats = stats_proxy_->GetStats();
1374 EXPECT_FALSE(stats.cpu_limited_resolution);
1375 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001376 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001377
mflodmancc3d4422017-08-03 08:27:51 -07001378 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001379}
1380
mflodmancc3d4422017-08-03 08:27:51 -07001381TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001382 video_stream_encoder_->OnBitrateUpdated(
1383 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001384
asaperssonfab67072017-04-04 05:51:49 -07001385 const int kWidth = 1280;
1386 const int kHeight = 720;
1387 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001388 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001389 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001390 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001391 EXPECT_FALSE(stats.cpu_limited_resolution);
1392 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1393
asaperssonfab67072017-04-04 05:51:49 -07001394 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001395 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001396 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001397 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001398 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001399 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001400 EXPECT_TRUE(stats.cpu_limited_resolution);
1401 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1402
1403 // Set new source with adaptation still enabled.
1404 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001405 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001406 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001407
asaperssonfab67072017-04-04 05:51:49 -07001408 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001409 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001410 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001411 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001412 EXPECT_TRUE(stats.cpu_limited_resolution);
1413 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1414
1415 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001416 video_stream_encoder_->SetSource(&new_video_source,
1417 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001418
asaperssonfab67072017-04-04 05:51:49 -07001419 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001420 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001421 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001422 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001423 EXPECT_FALSE(stats.cpu_limited_resolution);
1424 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1425
1426 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001427 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001428 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001429
asaperssonfab67072017-04-04 05:51:49 -07001430 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001431 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001432 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001433 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001434 EXPECT_TRUE(stats.cpu_limited_resolution);
1435 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1436
asaperssonfab67072017-04-04 05:51:49 -07001437 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001438 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001439 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001440 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001441 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001442 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001443 EXPECT_FALSE(stats.cpu_limited_resolution);
1444 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001445 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001446
mflodmancc3d4422017-08-03 08:27:51 -07001447 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001448}
1449
mflodmancc3d4422017-08-03 08:27:51 -07001450TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001451 video_stream_encoder_->OnBitrateUpdated(
1452 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001453
asaperssonfab67072017-04-04 05:51:49 -07001454 const int kWidth = 1280;
1455 const int kHeight = 720;
1456 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001457 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001458 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001459 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001460 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001461 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001462
1463 // Set new source with adaptation still enabled.
1464 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001465 video_stream_encoder_->SetSource(&new_video_source,
1466 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001467
asaperssonfab67072017-04-04 05:51:49 -07001468 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001469 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001470 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001471 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001472 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001473 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001474
asaperssonfab67072017-04-04 05:51:49 -07001475 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001476 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001477 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001478 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001479 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001480 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001481 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001482 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001483
asaperssonfab67072017-04-04 05:51:49 -07001484 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001485 video_stream_encoder_->SetSource(&new_video_source,
1486 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001487
asaperssonfab67072017-04-04 05:51:49 -07001488 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001489 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001490 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001491 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001492 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001493 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001494
asapersson02465b82017-04-10 01:12:52 -07001495 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001496 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001497 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001498
asaperssonfab67072017-04-04 05:51:49 -07001499 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001500 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001501 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001502 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001503 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001504 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1505 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001506
mflodmancc3d4422017-08-03 08:27:51 -07001507 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001508}
1509
mflodmancc3d4422017-08-03 08:27:51 -07001510TEST_F(VideoStreamEncoderTest,
1511 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001512 video_stream_encoder_->OnBitrateUpdated(
1513 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001514
1515 const int kWidth = 1280;
1516 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001517 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001518 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001519 video_source_.IncomingCapturedFrame(
1520 CreateFrame(timestamp_ms, kWidth, kHeight));
1521 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001522 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1524 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1525
1526 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001527 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001528 timestamp_ms += kFrameIntervalMs;
1529 video_source_.IncomingCapturedFrame(
1530 CreateFrame(timestamp_ms, kWidth, kHeight));
1531 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1534 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1535
1536 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001537 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001538 timestamp_ms += kFrameIntervalMs;
1539 video_source_.IncomingCapturedFrame(
1540 CreateFrame(timestamp_ms, kWidth, kHeight));
1541 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001542 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1544 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1545
Niels Möller4db138e2018-04-19 09:04:13 +02001546 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001547 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001548
1549 VideoEncoderConfig video_encoder_config;
1550 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1551 // Make format different, to force recreation of encoder.
1552 video_encoder_config.video_format.parameters["foo"] = "foo";
1553 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001554 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001555 timestamp_ms += kFrameIntervalMs;
1556 video_source_.IncomingCapturedFrame(
1557 CreateFrame(timestamp_ms, kWidth, kHeight));
1558 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001559 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1561 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1562
mflodmancc3d4422017-08-03 08:27:51 -07001563 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001564}
1565
mflodmancc3d4422017-08-03 08:27:51 -07001566TEST_F(VideoStreamEncoderTest,
1567 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001568 video_stream_encoder_->OnBitrateUpdated(
1569 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001570
asapersson0944a802017-04-07 00:57:58 -07001571 const int kWidth = 1280;
1572 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001573 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001574
asaperssonfab67072017-04-04 05:51:49 -07001575 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001576 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001577 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001578 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001579 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001580 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1581
asapersson02465b82017-04-10 01:12:52 -07001582 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001583 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001584 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001585 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001586 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001587 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001588 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001589 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1590
1591 // Set new source with adaptation still enabled.
1592 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001593 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001594 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001595
1596 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001597 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001598 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001599 stats = stats_proxy_->GetStats();
1600 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001601 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001602 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1603
sprangc5d62e22017-04-02 23:53:04 -07001604 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001605 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001606 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001607 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001608 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001609 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001610 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001611 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001612 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001613 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001614 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1615
sprangc5d62e22017-04-02 23:53:04 -07001616 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001617 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001618 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1619 mock_stats.input_frame_rate = 30;
1620 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001621 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001622 stats_proxy_->ResetMockStats();
1623
1624 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001625 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001626 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001627
1628 // Framerate now adapted.
1629 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001630 EXPECT_FALSE(stats.cpu_limited_resolution);
1631 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001632 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1633
1634 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001635 video_stream_encoder_->SetSource(&new_video_source,
1636 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001637 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001638 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001639 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001640
1641 stats = stats_proxy_->GetStats();
1642 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001643 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001644 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1645
1646 // Try to trigger overuse. Should not succeed.
1647 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001648 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001649 stats_proxy_->ResetMockStats();
1650
1651 stats = stats_proxy_->GetStats();
1652 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001653 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001654 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1655
1656 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001657 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001658 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001659 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001660 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001661 stats = stats_proxy_->GetStats();
1662 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001663 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001664 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001665
1666 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001667 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001668 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001669 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001670 stats = stats_proxy_->GetStats();
1671 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001672 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001673 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1674
1675 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001676 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001677 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001678 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001679 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001680 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001681 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001682 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001683 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001684 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001685 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1686
1687 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001688 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001689 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001690 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001691 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001692 stats = stats_proxy_->GetStats();
1693 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001694 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001695 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001696 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001697
mflodmancc3d4422017-08-03 08:27:51 -07001698 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001699}
1700
mflodmancc3d4422017-08-03 08:27:51 -07001701TEST_F(VideoStreamEncoderTest,
1702 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001703 const int kWidth = 1280;
1704 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001705 video_stream_encoder_->OnBitrateUpdated(
1706 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001707
asaperssonfab67072017-04-04 05:51:49 -07001708 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001709 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001710
asaperssonfab67072017-04-04 05:51:49 -07001711 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001712 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001713
asaperssonfab67072017-04-04 05:51:49 -07001714 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001715 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001716
asaperssonfab67072017-04-04 05:51:49 -07001717 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001718 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001719
kthelgason876222f2016-11-29 01:44:11 -08001720 // Expect a scale down.
1721 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001722 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001723
asapersson02465b82017-04-10 01:12:52 -07001724 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001725 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001726 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001727 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001728
asaperssonfab67072017-04-04 05:51:49 -07001729 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001730 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001731 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001732 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001733
asaperssonfab67072017-04-04 05:51:49 -07001734 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001735 EXPECT_EQ(std::numeric_limits<int>::max(),
1736 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001737
asaperssonfab67072017-04-04 05:51:49 -07001738 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001739 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001740 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001741 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001742
asapersson02465b82017-04-10 01:12:52 -07001743 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001744 EXPECT_EQ(std::numeric_limits<int>::max(),
1745 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001746
mflodmancc3d4422017-08-03 08:27:51 -07001747 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001748}
1749
mflodmancc3d4422017-08-03 08:27:51 -07001750TEST_F(VideoStreamEncoderTest,
1751 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001752 const int kWidth = 1280;
1753 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001754 video_stream_encoder_->OnBitrateUpdated(
1755 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001756
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001757 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001758 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001759 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001760 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001761
1762 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001763 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001764 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001765 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1766 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1767
1768 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001769 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001770 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001771 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1772 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1773 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1774
1775 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001776 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001777 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1778 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1779 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1780
mflodmancc3d4422017-08-03 08:27:51 -07001781 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001782}
1783
mflodmancc3d4422017-08-03 08:27:51 -07001784TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001785 const int kWidth = 1280;
1786 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001787 video_stream_encoder_->OnBitrateUpdated(
1788 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001789
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001790 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001791 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001792 video_stream_encoder_->SetSource(&source,
1793 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001794 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1795 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001796 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001797
1798 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001799 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001800 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1801 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1802 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1803 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1804
1805 // Trigger adapt down for same input resolution, expect no change.
1806 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1807 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001808 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001809 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1810 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1811 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1812
1813 // Trigger adapt down for larger input resolution, expect no change.
1814 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1815 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001816 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001817 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1818 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1819 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1820
mflodmancc3d4422017-08-03 08:27:51 -07001821 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001822}
1823
mflodmancc3d4422017-08-03 08:27:51 -07001824TEST_F(VideoStreamEncoderTest,
1825 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001826 const int kWidth = 1280;
1827 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001828 video_stream_encoder_->OnBitrateUpdated(
1829 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001830
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001831 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001832 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001833 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001834 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001835
1836 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001837 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001838 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001839 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1840 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1841
1842 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001843 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001844 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1846 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1847
mflodmancc3d4422017-08-03 08:27:51 -07001848 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001849}
1850
mflodmancc3d4422017-08-03 08:27:51 -07001851TEST_F(VideoStreamEncoderTest,
1852 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001853 const int kWidth = 1280;
1854 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001855 video_stream_encoder_->OnBitrateUpdated(
1856 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001857
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001858 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001859 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001860 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001861 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001862
1863 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001864 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001865 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001866 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001867 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1868
1869 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001870 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001871 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001872 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001873 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1874
mflodmancc3d4422017-08-03 08:27:51 -07001875 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001876}
1877
mflodmancc3d4422017-08-03 08:27:51 -07001878TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001879 const int kWidth = 1280;
1880 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001881 video_stream_encoder_->OnBitrateUpdated(
1882 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001883
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001884 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001885 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001886 video_stream_encoder_->SetSource(&source,
1887 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001888
1889 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1890 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001891 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001892 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1893 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1894 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1895
1896 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001897 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001898 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001899 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1900 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1901 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1902
mflodmancc3d4422017-08-03 08:27:51 -07001903 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001904}
1905
mflodmancc3d4422017-08-03 08:27:51 -07001906TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001907 const int kWidth = 1280;
1908 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001909 video_stream_encoder_->OnBitrateUpdated(
1910 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001911
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001912 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001913 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001914 video_stream_encoder_->SetSource(&source,
1915 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001916
1917 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1918 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001919 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1922 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1923
1924 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001925 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001926 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001927 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1928 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1929 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1930
mflodmancc3d4422017-08-03 08:27:51 -07001931 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001932}
1933
mflodmancc3d4422017-08-03 08:27:51 -07001934TEST_F(VideoStreamEncoderTest,
1935 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001936 const int kWidth = 1280;
1937 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02001938 video_stream_encoder_->OnBitrateUpdated(
1939 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001940
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001941 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001942 AdaptingFrameForwarder source;
1943 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001944 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001945 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001946
1947 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001948 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001949 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001950 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1951 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1952
1953 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001954 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001955 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001956 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001957 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001958 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1959 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1960
1961 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001962 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001963 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001964 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1965 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1966 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1967
mflodmancc3d4422017-08-03 08:27:51 -07001968 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001969}
1970
mflodmancc3d4422017-08-03 08:27:51 -07001971TEST_F(VideoStreamEncoderTest,
1972 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001973 const int kWidth = 1280;
1974 const int kHeight = 720;
1975 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02001976 video_stream_encoder_->OnBitrateUpdated(
1977 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001978
1979 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1980 stats.input_frame_rate = kInputFps;
1981 stats_proxy_->SetMockStats(stats);
1982
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001983 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001984 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1985 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001986 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001987
1988 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001989 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001990 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1991 sink_.WaitForEncodedFrame(2);
1992 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1993
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001994 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001995 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001996 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001997 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001998 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001999
2000 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002001 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002002 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2003 sink_.WaitForEncodedFrame(3);
2004 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2005
2006 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002007 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002008 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002009
mflodmancc3d4422017-08-03 08:27:51 -07002010 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002011}
2012
mflodmancc3d4422017-08-03 08:27:51 -07002013TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002014 const int kWidth = 1280;
2015 const int kHeight = 720;
2016 const size_t kNumFrames = 10;
2017
Erik Språng4c6ca302019-04-08 15:14:01 +02002018 video_stream_encoder_->OnBitrateUpdated(
2019 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002020
asaperssond0de2952017-04-21 01:47:31 -07002021 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002022 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002023 video_source_.set_adaptation_enabled(true);
2024
2025 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2026 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2027
2028 int downscales = 0;
2029 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002030 video_source_.IncomingCapturedFrame(
2031 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2032 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002033
asaperssonfab67072017-04-04 05:51:49 -07002034 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002035 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002036 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002037 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002038
2039 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2040 ++downscales;
2041
2042 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2043 EXPECT_EQ(downscales,
2044 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2045 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002046 }
mflodmancc3d4422017-08-03 08:27:51 -07002047 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002048}
2049
mflodmancc3d4422017-08-03 08:27:51 -07002050TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002051 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2052 const int kWidth = 1280;
2053 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002054 video_stream_encoder_->OnBitrateUpdated(
2055 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002056
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002057 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002058 AdaptingFrameForwarder source;
2059 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002060 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002061 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002062
Åsa Persson8c1bf952018-09-13 10:42:19 +02002063 int64_t timestamp_ms = kFrameIntervalMs;
2064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002065 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002066 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002067 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2068 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2069
2070 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002071 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002072 timestamp_ms += kFrameIntervalMs;
2073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2074 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002075 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002076 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2077 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2078
2079 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002080 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002081 timestamp_ms += kFrameIntervalMs;
2082 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002083 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002084 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002085 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2086 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2087
2088 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002089 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002090 timestamp_ms += kFrameIntervalMs;
2091 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2092 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002093 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002094 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2095 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2096
2097 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002098 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002099 timestamp_ms += kFrameIntervalMs;
2100 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002101 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002102 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002103 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2104 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2105
mflodmancc3d4422017-08-03 08:27:51 -07002106 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002107}
2108
mflodmancc3d4422017-08-03 08:27:51 -07002109TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002110 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2111 const int kWidth = 1280;
2112 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002113 video_stream_encoder_->OnBitrateUpdated(
2114 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002115
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002116 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002117 AdaptingFrameForwarder source;
2118 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002119 video_stream_encoder_->SetSource(&source,
2120 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002121
Åsa Persson8c1bf952018-09-13 10:42:19 +02002122 int64_t timestamp_ms = kFrameIntervalMs;
2123 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002124 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002125 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002126 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2127 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2128
2129 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002130 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002131 timestamp_ms += kFrameIntervalMs;
2132 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2133 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002134 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2135 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2136 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2137
2138 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002139 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002140 timestamp_ms += kFrameIntervalMs;
2141 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002142 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002143 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002144 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2145 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2146
2147 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002148 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002149 timestamp_ms += kFrameIntervalMs;
2150 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2151 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002152 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2153 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2154 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2155
2156 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002157 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002158 timestamp_ms += kFrameIntervalMs;
2159 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002160 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002161 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002162 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2163 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2164
mflodmancc3d4422017-08-03 08:27:51 -07002165 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002166}
2167
mflodmancc3d4422017-08-03 08:27:51 -07002168TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002169 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2170 const int kWidth = 1280;
2171 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002172 video_stream_encoder_->OnBitrateUpdated(
2173 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002174
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002175 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002176 AdaptingFrameForwarder source;
2177 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002178 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002179 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002180
Åsa Persson8c1bf952018-09-13 10:42:19 +02002181 int64_t timestamp_ms = kFrameIntervalMs;
2182 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002183 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002184 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002185 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2186 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2187 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2188 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2189
2190 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002191 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002192 timestamp_ms += kFrameIntervalMs;
2193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2194 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002195 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002196 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2197 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2198 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2199 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2200
2201 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002202 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002203 timestamp_ms += kFrameIntervalMs;
2204 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2205 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002206 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002207 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2209 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2210 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2211
Jonathan Yubc771b72017-12-08 17:04:29 -08002212 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002213 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002214 timestamp_ms += kFrameIntervalMs;
2215 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2216 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002217 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002218 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2219 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002220 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002221 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2222
Jonathan Yubc771b72017-12-08 17:04:29 -08002223 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002224 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002225 timestamp_ms += kFrameIntervalMs;
2226 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2227 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002228 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002229 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002230 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2231 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2232 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2233 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2234
Jonathan Yubc771b72017-12-08 17:04:29 -08002235 // Trigger quality adapt down, expect no change (min resolution reached).
2236 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002237 timestamp_ms += kFrameIntervalMs;
2238 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2239 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002240 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2241 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2242 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2243 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2244 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2245
2246 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002247 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002248 timestamp_ms += kFrameIntervalMs;
2249 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2250 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002251 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002252 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2253 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2254 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2255 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2256
2257 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2258 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002259 timestamp_ms += kFrameIntervalMs;
2260 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2261 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002262 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2263 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2264 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2265 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2266 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2267
2268 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2269 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002270 timestamp_ms += kFrameIntervalMs;
2271 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2272 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002273 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002274 last_wants = source.sink_wants();
2275 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002277 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002278 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2279
2280 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002281 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002282 timestamp_ms += kFrameIntervalMs;
2283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2284 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002285 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002286 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2287 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002288 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002289 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2290
2291 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002292 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002293 timestamp_ms += kFrameIntervalMs;
2294 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002295 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002296 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002297 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002298 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002300 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002301 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002302
mflodmancc3d4422017-08-03 08:27:51 -07002303 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002304}
2305
mflodmancc3d4422017-08-03 08:27:51 -07002306TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002307 const int kWidth = 640;
2308 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002309
Erik Språng4c6ca302019-04-08 15:14:01 +02002310 video_stream_encoder_->OnBitrateUpdated(
2311 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002312
perkj803d97f2016-11-01 11:45:46 -07002313 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002314 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002315 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002316 }
2317
mflodmancc3d4422017-08-03 08:27:51 -07002318 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002319 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002320 video_source_.IncomingCapturedFrame(CreateFrame(
2321 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002322 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002323 }
2324
mflodmancc3d4422017-08-03 08:27:51 -07002325 video_stream_encoder_->Stop();
2326 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002327 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002328
perkj803d97f2016-11-01 11:45:46 -07002329 EXPECT_EQ(1,
2330 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2331 EXPECT_EQ(
2332 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2333}
2334
mflodmancc3d4422017-08-03 08:27:51 -07002335TEST_F(VideoStreamEncoderTest,
2336 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002337 video_stream_encoder_->OnBitrateUpdated(
2338 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002339 const int kWidth = 640;
2340 const int kHeight = 360;
2341
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002342 video_stream_encoder_->SetSource(&video_source_,
2343 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002344
2345 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2346 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002347 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002348 }
2349
mflodmancc3d4422017-08-03 08:27:51 -07002350 video_stream_encoder_->Stop();
2351 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002352 stats_proxy_.reset();
2353
2354 EXPECT_EQ(0,
2355 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2356}
2357
mflodmancc3d4422017-08-03 08:27:51 -07002358TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002359 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002360 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002361
2362 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002363 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002364 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002365 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002366
sprang57c2fff2017-01-16 06:24:02 -08002367 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002368 .Times(1);
Erik Språng610c7632019-03-06 15:37:33 +01002369 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002370 DataRate::bps(kLowTargetBitrateBps),
2371 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002372
sprang57c2fff2017-01-16 06:24:02 -08002373 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002374 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2375 WaitForEncodedFrame(rtc::TimeMillis());
2376 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2377 fake_encoder_.GetAndResetLastBitrateAllocation();
2378 // Check that encoder has been updated too, not just allocation observer.
2379 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02002380 // TODO(srte): The use of millisecs here looks like an error, but the tests
2381 // fails using seconds, this should be investigated.
2382 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002383
2384 // Not called on second frame.
2385 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2386 .Times(0);
2387 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002388 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2389 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002390 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002391
2392 // Called after a process interval.
2393 const int64_t kProcessIntervalMs =
2394 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002395 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2396 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002397 const int64_t start_time_ms = rtc::TimeMillis();
2398 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2399 video_source_.IncomingCapturedFrame(
2400 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2401 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002402 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01002403 }
2404
2405 // Since rates are unchanged, encoder should not be reconfigured.
2406 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002407
mflodmancc3d4422017-08-03 08:27:51 -07002408 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002409}
2410
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01002411TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
2412 // 2 TLs configured, temporal layers supported by encoder.
2413 const int kNumTemporalLayers = 2;
2414 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
2415 fake_encoder_.SetTemporalLayersSupported(0, true);
2416
2417 // Bitrate allocated across temporal layers.
2418 const int kTl0Bps = kTargetBitrateBps *
2419 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2420 kNumTemporalLayers, /*temporal_id*/ 0);
2421 const int kTl1Bps = kTargetBitrateBps *
2422 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2423 kNumTemporalLayers, /*temporal_id*/ 1);
2424 VideoBitrateAllocation expected_bitrate;
2425 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
2426 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
2427
2428 VerifyAllocatedBitrate(expected_bitrate);
2429 video_stream_encoder_->Stop();
2430}
2431
2432TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
2433 // 2 TLs configured, temporal layers not supported by encoder.
2434 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2435 fake_encoder_.SetTemporalLayersSupported(0, false);
2436
2437 // Temporal layers not supported by the encoder.
2438 // Total bitrate should be at ti:0.
2439 VideoBitrateAllocation expected_bitrate;
2440 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
2441
2442 VerifyAllocatedBitrate(expected_bitrate);
2443 video_stream_encoder_->Stop();
2444}
2445
2446TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
2447 // 2 TLs configured, temporal layers only supported for first stream.
2448 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2449 fake_encoder_.SetTemporalLayersSupported(0, true);
2450 fake_encoder_.SetTemporalLayersSupported(1, false);
2451
2452 const int kS0Bps = 150000;
2453 const int kS0Tl0Bps =
2454 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2455 /*num_layers*/ 2, /*temporal_id*/ 0);
2456 const int kS0Tl1Bps =
2457 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2458 /*num_layers*/ 2, /*temporal_id*/ 1);
2459 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
2460 // Temporal layers not supported by si:1.
2461 VideoBitrateAllocation expected_bitrate;
2462 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
2463 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
2464 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
2465
2466 VerifyAllocatedBitrate(expected_bitrate);
2467 video_stream_encoder_->Stop();
2468}
2469
Niels Möller7dc26b72017-12-06 10:27:48 +01002470TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2471 const int kFrameWidth = 1280;
2472 const int kFrameHeight = 720;
2473 const int kFramerate = 24;
2474
Erik Språng4c6ca302019-04-08 15:14:01 +02002475 video_stream_encoder_->OnBitrateUpdated(
2476 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002477 test::FrameForwarder source;
2478 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002479 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002480
2481 // Insert a single frame, triggering initial configuration.
2482 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2483 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2484
2485 EXPECT_EQ(
2486 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2487 kDefaultFramerate);
2488
2489 // Trigger reconfigure encoder (without resetting the entire instance).
2490 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002491 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002492 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2493 video_encoder_config.number_of_streams = 1;
2494 video_encoder_config.video_stream_factory =
2495 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2496 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002497 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002498 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2499
2500 // Detector should be updated with fps limit from codec config.
2501 EXPECT_EQ(
2502 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2503 kFramerate);
2504
2505 // Trigger overuse, max framerate should be reduced.
2506 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2507 stats.input_frame_rate = kFramerate;
2508 stats_proxy_->SetMockStats(stats);
2509 video_stream_encoder_->TriggerCpuOveruse();
2510 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2511 int adapted_framerate =
2512 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2513 EXPECT_LT(adapted_framerate, kFramerate);
2514
2515 // Trigger underuse, max framerate should go back to codec configured fps.
2516 // Set extra low fps, to make sure it's actually reset, not just incremented.
2517 stats = stats_proxy_->GetStats();
2518 stats.input_frame_rate = adapted_framerate / 2;
2519 stats_proxy_->SetMockStats(stats);
2520 video_stream_encoder_->TriggerCpuNormalUsage();
2521 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2522 EXPECT_EQ(
2523 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2524 kFramerate);
2525
2526 video_stream_encoder_->Stop();
2527}
2528
2529TEST_F(VideoStreamEncoderTest,
2530 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2531 const int kFrameWidth = 1280;
2532 const int kFrameHeight = 720;
2533 const int kLowFramerate = 15;
2534 const int kHighFramerate = 25;
2535
Erik Språng4c6ca302019-04-08 15:14:01 +02002536 video_stream_encoder_->OnBitrateUpdated(
2537 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002538 test::FrameForwarder source;
2539 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002540 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002541
2542 // Trigger initial configuration.
2543 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002544 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002545 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2546 video_encoder_config.number_of_streams = 1;
2547 video_encoder_config.video_stream_factory =
2548 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2549 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2550 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002551 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002552 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2553
2554 EXPECT_EQ(
2555 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2556 kLowFramerate);
2557
2558 // Trigger overuse, max framerate should be reduced.
2559 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2560 stats.input_frame_rate = kLowFramerate;
2561 stats_proxy_->SetMockStats(stats);
2562 video_stream_encoder_->TriggerCpuOveruse();
2563 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2564 int adapted_framerate =
2565 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2566 EXPECT_LT(adapted_framerate, kLowFramerate);
2567
2568 // Reconfigure the encoder with a new (higher max framerate), max fps should
2569 // still respect the adaptation.
2570 video_encoder_config.video_stream_factory =
2571 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2572 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2573 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002574 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2576
2577 EXPECT_EQ(
2578 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2579 adapted_framerate);
2580
2581 // Trigger underuse, max framerate should go back to codec configured fps.
2582 stats = stats_proxy_->GetStats();
2583 stats.input_frame_rate = adapted_framerate;
2584 stats_proxy_->SetMockStats(stats);
2585 video_stream_encoder_->TriggerCpuNormalUsage();
2586 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2587 EXPECT_EQ(
2588 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2589 kHighFramerate);
2590
2591 video_stream_encoder_->Stop();
2592}
2593
mflodmancc3d4422017-08-03 08:27:51 -07002594TEST_F(VideoStreamEncoderTest,
2595 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002596 const int kFrameWidth = 1280;
2597 const int kFrameHeight = 720;
2598 const int kFramerate = 24;
2599
Erik Språng4c6ca302019-04-08 15:14:01 +02002600 video_stream_encoder_->OnBitrateUpdated(
2601 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002602 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002603 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002604 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002605
2606 // Trigger initial configuration.
2607 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002608 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002609 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2610 video_encoder_config.number_of_streams = 1;
2611 video_encoder_config.video_stream_factory =
2612 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2613 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002614 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002615 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002616 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002617
Niels Möller7dc26b72017-12-06 10:27:48 +01002618 EXPECT_EQ(
2619 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2620 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002621
2622 // Trigger overuse, max framerate should be reduced.
2623 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2624 stats.input_frame_rate = kFramerate;
2625 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002626 video_stream_encoder_->TriggerCpuOveruse();
2627 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002628 int adapted_framerate =
2629 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002630 EXPECT_LT(adapted_framerate, kFramerate);
2631
2632 // Change degradation preference to not enable framerate scaling. Target
2633 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002634 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002635 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002636 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002637 EXPECT_EQ(
2638 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2639 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002640
mflodmancc3d4422017-08-03 08:27:51 -07002641 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002642}
2643
mflodmancc3d4422017-08-03 08:27:51 -07002644TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002645 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01002646 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002647 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2648 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002649 const int kWidth = 640;
2650 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002651
asaperssonfab67072017-04-04 05:51:49 -07002652 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002653
2654 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002655 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002656
2657 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002658 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002659
sprangc5d62e22017-04-02 23:53:04 -07002660 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002661
asaperssonfab67072017-04-04 05:51:49 -07002662 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002663 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002664 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002665
2666 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002667 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002668
sprangc5d62e22017-04-02 23:53:04 -07002669 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002670
mflodmancc3d4422017-08-03 08:27:51 -07002671 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002672}
2673
mflodmancc3d4422017-08-03 08:27:51 -07002674TEST_F(VideoStreamEncoderTest,
2675 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002676 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01002677 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002678 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2679 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002680 const int kWidth = 640;
2681 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002682
2683 // We expect the n initial frames to get dropped.
2684 int i;
2685 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002686 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002687 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002688 }
2689 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002690 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002691 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002692
2693 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002694 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002695
mflodmancc3d4422017-08-03 08:27:51 -07002696 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002697}
2698
mflodmancc3d4422017-08-03 08:27:51 -07002699TEST_F(VideoStreamEncoderTest,
2700 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002701 const int kWidth = 640;
2702 const int kHeight = 360;
Erik Språng610c7632019-03-06 15:37:33 +01002703 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002704 DataRate::bps(kLowTargetBitrateBps),
2705 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002706
2707 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002709 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002710
asaperssonfab67072017-04-04 05:51:49 -07002711 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002712 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002713 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002714
mflodmancc3d4422017-08-03 08:27:51 -07002715 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002716}
2717
mflodmancc3d4422017-08-03 08:27:51 -07002718TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002719 const int kWidth = 640;
2720 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002721 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002722
2723 VideoEncoderConfig video_encoder_config;
2724 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2725 // Make format different, to force recreation of encoder.
2726 video_encoder_config.video_format.parameters["foo"] = "foo";
2727 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002728 kMaxPayloadLength);
Erik Språng610c7632019-03-06 15:37:33 +01002729 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002730 DataRate::bps(kLowTargetBitrateBps),
2731 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002732
kthelgasonb83797b2017-02-14 11:57:25 -08002733 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002734 video_stream_encoder_->SetSource(&video_source_,
2735 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002736
asaperssonfab67072017-04-04 05:51:49 -07002737 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002738 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002739 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002740
mflodmancc3d4422017-08-03 08:27:51 -07002741 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002742 fake_encoder_.SetQualityScaling(true);
2743}
2744
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002745TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2746 webrtc::test::ScopedFieldTrials field_trials(
2747 "WebRTC-InitialFramedrop/Enabled/");
2748 // Reset encoder for field trials to take effect.
2749 ConfigureEncoder(video_encoder_config_.Copy());
2750 const int kTooLowBitrateForFrameSizeBps = 10000;
2751 const int kWidth = 640;
2752 const int kHeight = 360;
2753
Erik Språng4c6ca302019-04-08 15:14:01 +02002754 video_stream_encoder_->OnBitrateUpdated(
2755 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002756 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2757 // Frame should not be dropped.
2758 WaitForEncodedFrame(1);
2759
Erik Språng610c7632019-03-06 15:37:33 +01002760 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02002761 DataRate::bps(kTooLowBitrateForFrameSizeBps),
2762 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002763 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2764 // Expect to drop this frame, the wait should time out.
2765 ExpectDroppedFrame();
2766
2767 // Expect the sink_wants to specify a scaled frame.
2768 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2769 video_stream_encoder_->Stop();
2770}
2771
mflodmancc3d4422017-08-03 08:27:51 -07002772TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002773 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2774 const int kTooSmallWidth = 10;
2775 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02002776 video_stream_encoder_->OnBitrateUpdated(
2777 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002778
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002779 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002780 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002781 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002782 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002783 VerifyNoLimitation(source.sink_wants());
2784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2785
2786 // Trigger adapt down, too small frame, expect no change.
2787 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002788 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002789 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002790 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2793
mflodmancc3d4422017-08-03 08:27:51 -07002794 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002795}
2796
mflodmancc3d4422017-08-03 08:27:51 -07002797TEST_F(VideoStreamEncoderTest,
2798 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002799 const int kTooSmallWidth = 10;
2800 const int kTooSmallHeight = 10;
2801 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02002802 video_stream_encoder_->OnBitrateUpdated(
2803 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002804
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002805 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002806 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002807 video_stream_encoder_->SetSource(&source,
2808 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002809 VerifyNoLimitation(source.sink_wants());
2810 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2811 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2812
2813 // Trigger adapt down, expect limited framerate.
2814 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002815 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002816 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002817 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2818 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2819 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2820 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2821
2822 // Trigger adapt down, too small frame, expect no change.
2823 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002824 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002825 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002826 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2828 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2829 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2830
mflodmancc3d4422017-08-03 08:27:51 -07002831 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002832}
2833
mflodmancc3d4422017-08-03 08:27:51 -07002834TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002835 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02002836 video_stream_encoder_->OnBitrateUpdated(
2837 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002838 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002839 const int kFrameWidth = 1280;
2840 const int kFrameHeight = 720;
2841 video_source_.IncomingCapturedFrame(
2842 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002843 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002844 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002845}
2846
sprangb1ca0732017-02-01 08:38:12 -08002847// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002848TEST_F(VideoStreamEncoderTest,
2849 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002850 video_stream_encoder_->OnBitrateUpdated(
2851 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002852
2853 const int kFrameWidth = 1280;
2854 const int kFrameHeight = 720;
2855 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002856 // requested by
2857 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002858 video_source_.set_adaptation_enabled(true);
2859
2860 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002861 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002862 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002863
2864 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002865 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002866 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002867 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002868 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002869
asaperssonfab67072017-04-04 05:51:49 -07002870 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002871 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002872 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002873 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002874 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002875
mflodmancc3d4422017-08-03 08:27:51 -07002876 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002877}
sprangfe627f32017-03-29 08:24:59 -07002878
mflodmancc3d4422017-08-03 08:27:51 -07002879TEST_F(VideoStreamEncoderTest,
2880 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002881 const int kFrameWidth = 1280;
2882 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002883
Erik Språng4c6ca302019-04-08 15:14:01 +02002884 video_stream_encoder_->OnBitrateUpdated(
2885 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07002886 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002887 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002888 video_source_.set_adaptation_enabled(true);
2889
sprang4847ae62017-06-27 07:06:52 -07002890 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002891
2892 video_source_.IncomingCapturedFrame(
2893 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002894 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002895
2896 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002897 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002898
2899 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002900 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002901 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002902 video_source_.IncomingCapturedFrame(
2903 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002904 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002905 }
2906
2907 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002908 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002909 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002910 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002911 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002912 video_source_.IncomingCapturedFrame(
2913 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002914 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002915 ++num_frames_dropped;
2916 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002917 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002918 }
2919 }
2920
sprang4847ae62017-06-27 07:06:52 -07002921 // Add some slack to account for frames dropped by the frame dropper.
2922 const int kErrorMargin = 1;
2923 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002924 kErrorMargin);
2925
2926 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002927 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002928 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002929 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002930 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002931 video_source_.IncomingCapturedFrame(
2932 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002933 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002934 ++num_frames_dropped;
2935 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002936 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002937 }
2938 }
sprang4847ae62017-06-27 07:06:52 -07002939 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002940 kErrorMargin);
2941
2942 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002943 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002944 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002945 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002946 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002947 video_source_.IncomingCapturedFrame(
2948 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002949 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002950 ++num_frames_dropped;
2951 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002952 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002953 }
2954 }
sprang4847ae62017-06-27 07:06:52 -07002955 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002956 kErrorMargin);
2957
2958 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002959 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002960 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002961 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002962 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002963 video_source_.IncomingCapturedFrame(
2964 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002965 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002966 ++num_frames_dropped;
2967 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002968 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002969 }
2970 }
2971 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2972
mflodmancc3d4422017-08-03 08:27:51 -07002973 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002974}
2975
mflodmancc3d4422017-08-03 08:27:51 -07002976TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002977 const int kFramerateFps = 5;
2978 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002979 const int kFrameWidth = 1280;
2980 const int kFrameHeight = 720;
2981
sprang4847ae62017-06-27 07:06:52 -07002982 // Reconfigure encoder with two temporal layers and screensharing, which will
2983 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002984 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002985
Erik Språng4c6ca302019-04-08 15:14:01 +02002986 video_stream_encoder_->OnBitrateUpdated(
2987 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07002988 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002989 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002990 video_source_.set_adaptation_enabled(true);
2991
sprang4847ae62017-06-27 07:06:52 -07002992 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002993
2994 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002995 rtc::VideoSinkWants last_wants;
2996 do {
2997 last_wants = video_source_.sink_wants();
2998
sprangc5d62e22017-04-02 23:53:04 -07002999 // Insert frames to get a new fps estimate...
3000 for (int j = 0; j < kFramerateFps; ++j) {
3001 video_source_.IncomingCapturedFrame(
3002 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08003003 if (video_source_.last_sent_width()) {
3004 sink_.WaitForEncodedFrame(timestamp_ms);
3005 }
sprangc5d62e22017-04-02 23:53:04 -07003006 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003007 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07003008 }
3009 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07003010 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003011 } while (video_source_.sink_wants().max_framerate_fps <
3012 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003013
Jonathan Yubc771b72017-12-08 17:04:29 -08003014 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003015
mflodmancc3d4422017-08-03 08:27:51 -07003016 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003017}
asaperssonf7e294d2017-06-13 23:25:22 -07003018
mflodmancc3d4422017-08-03 08:27:51 -07003019TEST_F(VideoStreamEncoderTest,
3020 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003021 const int kWidth = 1280;
3022 const int kHeight = 720;
3023 const int64_t kFrameIntervalMs = 150;
3024 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003025 video_stream_encoder_->OnBitrateUpdated(
3026 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003027
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003028 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003029 AdaptingFrameForwarder source;
3030 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003031 video_stream_encoder_->SetSource(&source,
3032 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003033 timestamp_ms += kFrameIntervalMs;
3034 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003035 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003036 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003037 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3038 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3039 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3040
3041 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003042 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003043 timestamp_ms += kFrameIntervalMs;
3044 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003045 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003046 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3047 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3048 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3049 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3050
3051 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003052 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003053 timestamp_ms += kFrameIntervalMs;
3054 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003055 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003056 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3057 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3058 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3059 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3060
3061 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003062 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003063 timestamp_ms += kFrameIntervalMs;
3064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003065 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003066 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3067 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3068 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3069 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3070
3071 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003072 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003073 timestamp_ms += kFrameIntervalMs;
3074 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003075 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003076 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3077 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3078 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3079 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3080
3081 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003082 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003083 timestamp_ms += kFrameIntervalMs;
3084 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003085 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003086 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3087 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3088 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3089 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3090
3091 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003092 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003093 timestamp_ms += kFrameIntervalMs;
3094 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003095 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003096 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3097 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3098 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3099 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3100
3101 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003102 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003103 timestamp_ms += kFrameIntervalMs;
3104 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003105 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003106 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3107 rtc::VideoSinkWants last_wants = source.sink_wants();
3108 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3109 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3110 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3111
3112 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003113 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003114 timestamp_ms += kFrameIntervalMs;
3115 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003116 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003117 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3118 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3119 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3120 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3121
3122 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003123 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003124 timestamp_ms += kFrameIntervalMs;
3125 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003126 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003127 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3128 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3129 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3130 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3131
3132 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003133 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003134 timestamp_ms += kFrameIntervalMs;
3135 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003136 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003137 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3138 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3139 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3140 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3141
3142 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003143 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003144 timestamp_ms += kFrameIntervalMs;
3145 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003146 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003147 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3148 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3149 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3150 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3151
3152 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003154 timestamp_ms += kFrameIntervalMs;
3155 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003156 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003157 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3158 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3159 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3160 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3161
3162 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003163 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003164 timestamp_ms += kFrameIntervalMs;
3165 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003166 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003167 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3168 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3169 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3170 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3171
3172 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003173 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003174 timestamp_ms += kFrameIntervalMs;
3175 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003176 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003177 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3179 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3180 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3181
3182 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003183 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003184 timestamp_ms += kFrameIntervalMs;
3185 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003186 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003187 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003188 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003189 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3190 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3191 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3192
3193 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003194 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003195 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003196 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3197
mflodmancc3d4422017-08-03 08:27:51 -07003198 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003199}
3200
mflodmancc3d4422017-08-03 08:27:51 -07003201TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003202 const int kWidth = 1280;
3203 const int kHeight = 720;
3204 const int64_t kFrameIntervalMs = 150;
3205 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003206 video_stream_encoder_->OnBitrateUpdated(
3207 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003208
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003209 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003210 AdaptingFrameForwarder source;
3211 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003212 video_stream_encoder_->SetSource(&source,
3213 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003214 timestamp_ms += kFrameIntervalMs;
3215 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003216 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003217 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003218 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3219 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3220 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3221 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3222 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3223 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3224
3225 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003226 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003227 timestamp_ms += kFrameIntervalMs;
3228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003229 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003230 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3231 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3232 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3233 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3234 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3235 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3236 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3237
3238 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003239 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003240 timestamp_ms += kFrameIntervalMs;
3241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003242 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003243 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3245 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3246 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3247 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3248 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3249 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3250
3251 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003252 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003253 timestamp_ms += kFrameIntervalMs;
3254 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003255 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003256 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3258 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3259 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3260 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3261 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3262 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3263
3264 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003265 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003266 timestamp_ms += kFrameIntervalMs;
3267 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003268 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003269 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3270 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3271 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3272 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3273 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3274 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3275 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3276
3277 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003278 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003279 timestamp_ms += kFrameIntervalMs;
3280 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003281 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003282 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3283 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3284 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3285 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3286 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3287 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3288 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3289
3290 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003291 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003292 timestamp_ms += kFrameIntervalMs;
3293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003294 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003295 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003296 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3299 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3300 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3301 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3302 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3303
3304 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003305 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003306 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003307 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3308 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3309
mflodmancc3d4422017-08-03 08:27:51 -07003310 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003311}
3312
mflodmancc3d4422017-08-03 08:27:51 -07003313TEST_F(VideoStreamEncoderTest,
3314 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003315 const int kWidth = 640;
3316 const int kHeight = 360;
3317 const int kFpsLimit = 15;
3318 const int64_t kFrameIntervalMs = 150;
3319 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003320 video_stream_encoder_->OnBitrateUpdated(
3321 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003322
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003323 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003324 AdaptingFrameForwarder source;
3325 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003326 video_stream_encoder_->SetSource(&source,
3327 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003328 timestamp_ms += kFrameIntervalMs;
3329 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003330 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003331 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003332 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3333 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3334 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3335 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3336 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3337 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3338
3339 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003340 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003341 timestamp_ms += kFrameIntervalMs;
3342 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003343 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003344 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3345 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3346 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3347 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3348 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3349 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3350 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351
3352 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003353 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003354 timestamp_ms += kFrameIntervalMs;
3355 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003356 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003357 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3359 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3360 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3361 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3362 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3363 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3364
3365 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003366 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003367 timestamp_ms += kFrameIntervalMs;
3368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003369 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003370 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3371 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3373 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3374 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3375 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3376 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3377
3378 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003380 timestamp_ms += kFrameIntervalMs;
3381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003382 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003383 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3385 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3386 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3387 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3388 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3389 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3390
3391 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003393 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003394 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3395 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3396
mflodmancc3d4422017-08-03 08:27:51 -07003397 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003398}
3399
mflodmancc3d4422017-08-03 08:27:51 -07003400TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003401 // Simulates simulcast behavior and makes highest stream resolutions divisible
3402 // by 4.
3403 class CroppingVideoStreamFactory
3404 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3405 public:
3406 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3407 int framerate)
3408 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3409 EXPECT_GT(num_temporal_layers, 0u);
3410 EXPECT_GT(framerate, 0);
3411 }
3412
3413 private:
3414 std::vector<VideoStream> CreateEncoderStreams(
3415 int width,
3416 int height,
3417 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003418 std::vector<VideoStream> streams = test::CreateVideoStreams(
3419 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003420 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003421 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003422 stream.max_framerate = framerate_;
3423 }
3424 return streams;
3425 }
3426
3427 const size_t num_temporal_layers_;
3428 const int framerate_;
3429 };
3430
3431 const int kFrameWidth = 1920;
3432 const int kFrameHeight = 1080;
3433 // 3/4 of 1920.
3434 const int kAdaptedFrameWidth = 1440;
3435 // 3/4 of 1080 rounded down to multiple of 4.
3436 const int kAdaptedFrameHeight = 808;
3437 const int kFramerate = 24;
3438
Erik Språng4c6ca302019-04-08 15:14:01 +02003439 video_stream_encoder_->OnBitrateUpdated(
3440 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003441 // Trigger reconfigure encoder (without resetting the entire instance).
3442 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003443 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003444 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3445 video_encoder_config.number_of_streams = 1;
3446 video_encoder_config.video_stream_factory =
3447 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003448 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003449 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003450 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003451
3452 video_source_.set_adaptation_enabled(true);
3453
3454 video_source_.IncomingCapturedFrame(
3455 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003456 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003457
3458 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003459 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003460 video_source_.IncomingCapturedFrame(
3461 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003462 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003463
mflodmancc3d4422017-08-03 08:27:51 -07003464 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003465}
3466
mflodmancc3d4422017-08-03 08:27:51 -07003467TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003468 const int kFrameWidth = 1280;
3469 const int kFrameHeight = 720;
3470 const int kLowFps = 2;
3471 const int kHighFps = 30;
3472
Erik Språng4c6ca302019-04-08 15:14:01 +02003473 video_stream_encoder_->OnBitrateUpdated(
3474 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003475
3476 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3477 max_framerate_ = kLowFps;
3478
3479 // Insert 2 seconds of 2fps video.
3480 for (int i = 0; i < kLowFps * 2; ++i) {
3481 video_source_.IncomingCapturedFrame(
3482 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3483 WaitForEncodedFrame(timestamp_ms);
3484 timestamp_ms += 1000 / kLowFps;
3485 }
3486
3487 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02003488 video_stream_encoder_->OnBitrateUpdated(
3489 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003490 video_source_.IncomingCapturedFrame(
3491 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3492 WaitForEncodedFrame(timestamp_ms);
3493 timestamp_ms += 1000 / kLowFps;
3494
3495 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3496
3497 // Insert 30fps frames for just a little more than the forced update period.
3498 const int kVcmTimerIntervalFrames =
3499 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3500 const int kFrameIntervalMs = 1000 / kHighFps;
3501 max_framerate_ = kHighFps;
3502 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3503 video_source_.IncomingCapturedFrame(
3504 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3505 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3506 // be dropped if the encoder hans't been updated with the new higher target
3507 // framerate yet, causing it to overshoot the target bitrate and then
3508 // suffering the wrath of the media optimizer.
3509 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3510 timestamp_ms += kFrameIntervalMs;
3511 }
3512
3513 // Don expect correct measurement just yet, but it should be higher than
3514 // before.
3515 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3516
mflodmancc3d4422017-08-03 08:27:51 -07003517 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003518}
3519
mflodmancc3d4422017-08-03 08:27:51 -07003520TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003521 const int kFrameWidth = 1280;
3522 const int kFrameHeight = 720;
3523 const int kTargetBitrateBps = 1000000;
3524
3525 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003526 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02003527 video_stream_encoder_->OnBitrateUpdated(
3528 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003529 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003530
3531 // Insert a first video frame, causes another bitrate update.
3532 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3533 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3534 video_source_.IncomingCapturedFrame(
3535 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3536 WaitForEncodedFrame(timestamp_ms);
3537
3538 // Next, simulate video suspension due to pacer queue overrun.
Erik Språng4c6ca302019-04-08 15:14:01 +02003539 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01003540 1);
sprang4847ae62017-06-27 07:06:52 -07003541
3542 // Skip ahead until a new periodic parameter update should have occured.
3543 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003544 fake_clock_.AdvanceTime(
3545 TimeDelta::ms(vcm::VCMProcessTimer::kDefaultProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07003546
3547 // Bitrate observer should not be called.
3548 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3549 video_source_.IncomingCapturedFrame(
3550 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3551 ExpectDroppedFrame();
3552
mflodmancc3d4422017-08-03 08:27:51 -07003553 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003554}
ilnik6b826ef2017-06-16 06:53:48 -07003555
Niels Möller4db138e2018-04-19 09:04:13 +02003556TEST_F(VideoStreamEncoderTest,
3557 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3558 const int kFrameWidth = 1280;
3559 const int kFrameHeight = 720;
3560 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02003561 video_stream_encoder_->OnBitrateUpdated(
3562 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02003563 video_source_.IncomingCapturedFrame(
3564 CreateFrame(1, kFrameWidth, kFrameHeight));
3565 WaitForEncodedFrame(1);
3566 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3567 .low_encode_usage_threshold_percent,
3568 default_options.low_encode_usage_threshold_percent);
3569 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3570 .high_encode_usage_threshold_percent,
3571 default_options.high_encode_usage_threshold_percent);
3572 video_stream_encoder_->Stop();
3573}
3574
3575TEST_F(VideoStreamEncoderTest,
3576 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3577 const int kFrameWidth = 1280;
3578 const int kFrameHeight = 720;
3579 CpuOveruseOptions hardware_options;
3580 hardware_options.low_encode_usage_threshold_percent = 150;
3581 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003582 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003583
Erik Språng4c6ca302019-04-08 15:14:01 +02003584 video_stream_encoder_->OnBitrateUpdated(
3585 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02003586 video_source_.IncomingCapturedFrame(
3587 CreateFrame(1, kFrameWidth, kFrameHeight));
3588 WaitForEncodedFrame(1);
3589 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3590 .low_encode_usage_threshold_percent,
3591 hardware_options.low_encode_usage_threshold_percent);
3592 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3593 .high_encode_usage_threshold_percent,
3594 hardware_options.high_encode_usage_threshold_percent);
3595 video_stream_encoder_->Stop();
3596}
3597
Niels Möller6bb5ab92019-01-11 11:11:10 +01003598TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3599 const int kFrameWidth = 320;
3600 const int kFrameHeight = 240;
3601 const int kFps = 30;
3602 const int kTargetBitrateBps = 120000;
3603 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3604
Erik Språng4c6ca302019-04-08 15:14:01 +02003605 video_stream_encoder_->OnBitrateUpdated(
3606 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003607
3608 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3609 max_framerate_ = kFps;
3610
3611 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3612 fake_encoder_.SimulateOvershoot(1.0);
3613 int num_dropped = 0;
3614 for (int i = 0; i < kNumFramesInRun; ++i) {
3615 video_source_.IncomingCapturedFrame(
3616 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3617 // Wait up to two frame durations for a frame to arrive.
3618 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3619 ++num_dropped;
3620 }
3621 timestamp_ms += 1000 / kFps;
3622 }
3623
Erik Språnga8d48ab2019-02-08 14:17:40 +01003624 // Framerate should be measured to be near the expected target rate.
3625 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3626
3627 // Frame drops should be within 5% of expected 0%.
3628 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003629
3630 // Make encoder produce frames at double the expected bitrate during 3 seconds
3631 // of video, verify number of drops. Rate needs to be slightly changed in
3632 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003633 double overshoot_factor = 2.0;
3634 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3635 // With bitrate adjuster, when need to overshoot even more to trigger
3636 // frame dropping.
3637 overshoot_factor *= 2;
3638 }
3639 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01003640 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003641 DataRate::bps(kTargetBitrateBps + 1000),
3642 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003643 num_dropped = 0;
3644 for (int i = 0; i < kNumFramesInRun; ++i) {
3645 video_source_.IncomingCapturedFrame(
3646 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3647 // Wait up to two frame durations for a frame to arrive.
3648 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3649 ++num_dropped;
3650 }
3651 timestamp_ms += 1000 / kFps;
3652 }
3653
Erik Språng4c6ca302019-04-08 15:14:01 +02003654 video_stream_encoder_->OnBitrateUpdated(
3655 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01003656
3657 // Target framerate should be still be near the expected target, despite
3658 // the frame drops.
3659 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3660
3661 // Frame drops should be within 5% of expected 50%.
3662 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003663
3664 video_stream_encoder_->Stop();
3665}
3666
3667TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3668 const int kFrameWidth = 320;
3669 const int kFrameHeight = 240;
3670 const int kActualInputFps = 24;
3671 const int kTargetBitrateBps = 120000;
3672
3673 ASSERT_GT(max_framerate_, kActualInputFps);
3674
3675 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3676 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02003677 video_stream_encoder_->OnBitrateUpdated(
3678 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003679
3680 // Insert 3 seconds of video, with an input fps lower than configured max.
3681 for (int i = 0; i < kActualInputFps * 3; ++i) {
3682 video_source_.IncomingCapturedFrame(
3683 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3684 // Wait up to two frame durations for a frame to arrive.
3685 WaitForEncodedFrame(timestamp_ms);
3686 timestamp_ms += 1000 / kActualInputFps;
3687 }
3688
3689 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3690
3691 video_stream_encoder_->Stop();
3692}
3693
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003694TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3695 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02003696 video_stream_encoder_->OnBitrateUpdated(
3697 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003698
3699 fake_encoder_.BlockNextEncode();
3700 video_source_.IncomingCapturedFrame(
3701 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3702 WaitForEncodedFrame(1);
3703 // On the very first frame full update should be forced.
3704 rect = fake_encoder_.GetLastUpdateRect();
3705 EXPECT_EQ(rect.offset_x, 0);
3706 EXPECT_EQ(rect.offset_y, 0);
3707 EXPECT_EQ(rect.height, codec_height_);
3708 EXPECT_EQ(rect.width, codec_width_);
3709 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3710 // call to ContinueEncode.
3711 video_source_.IncomingCapturedFrame(
3712 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3713 ExpectDroppedFrame();
3714 video_source_.IncomingCapturedFrame(
3715 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3716 ExpectDroppedFrame();
3717 fake_encoder_.ContinueEncode();
3718 WaitForEncodedFrame(3);
3719 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3720 rect = fake_encoder_.GetLastUpdateRect();
3721 EXPECT_EQ(rect.offset_x, 1);
3722 EXPECT_EQ(rect.offset_y, 0);
3723 EXPECT_EQ(rect.width, 10);
3724 EXPECT_EQ(rect.height, 1);
3725
3726 video_source_.IncomingCapturedFrame(
3727 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3728 WaitForEncodedFrame(4);
3729 // Previous frame was encoded, so no accumulation should happen.
3730 rect = fake_encoder_.GetLastUpdateRect();
3731 EXPECT_EQ(rect.offset_x, 0);
3732 EXPECT_EQ(rect.offset_y, 0);
3733 EXPECT_EQ(rect.width, 1);
3734 EXPECT_EQ(rect.height, 1);
3735
3736 video_stream_encoder_->Stop();
3737}
3738
Erik Språngd7329ca2019-02-21 21:19:53 +01003739TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003740 video_stream_encoder_->OnBitrateUpdated(
3741 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003742
3743 // First frame is always keyframe.
3744 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3745 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01003746 EXPECT_THAT(
3747 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003748 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003749
3750 // Insert delta frame.
3751 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3752 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01003753 EXPECT_THAT(
3754 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003755 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003756
3757 // Request next frame be a key-frame.
3758 video_stream_encoder_->SendKeyFrame();
3759 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3760 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01003761 EXPECT_THAT(
3762 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003763 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003764
3765 video_stream_encoder_->Stop();
3766}
3767
3768TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3769 // Setup simulcast with three streams.
3770 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01003771 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003772 DataRate::bps(kSimulcastTargetBitrateBps),
3773 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003774 // Wait for all three layers before triggering event.
3775 sink_.SetNumExpectedLayers(3);
3776
3777 // First frame is always keyframe.
3778 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3779 WaitForEncodedFrame(1);
3780 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003781 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
3782 VideoFrameType::kVideoFrameKey,
3783 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003784
3785 // Insert delta frame.
3786 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3787 WaitForEncodedFrame(2);
3788 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003789 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
3790 VideoFrameType::kVideoFrameDelta,
3791 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003792
3793 // Request next frame be a key-frame.
3794 // Only first stream is configured to produce key-frame.
3795 video_stream_encoder_->SendKeyFrame();
3796 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3797 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02003798
3799 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
3800 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01003801 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003802 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02003803 VideoFrameType::kVideoFrameKey,
3804 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003805
3806 video_stream_encoder_->Stop();
3807}
3808
3809TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3810 // Configure internal source factory and setup test again.
3811 encoder_factory_.SetHasInternalSource(true);
3812 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02003813 video_stream_encoder_->OnBitrateUpdated(
3814 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01003815
3816 // Call encoder directly, simulating internal source where encoded frame
3817 // callback in VideoStreamEncoder is called despite no OnFrame().
3818 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3819 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003820 EXPECT_THAT(
3821 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003822 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003823
Niels Möller8f7ce222019-03-21 15:43:58 +01003824 const std::vector<VideoFrameType> kDeltaFrame = {
3825 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01003826 // Need to set timestamp manually since manually for injected frame.
3827 VideoFrame frame = CreateFrame(101, nullptr);
3828 frame.set_timestamp(101);
3829 fake_encoder_.InjectFrame(frame, false);
3830 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003831 EXPECT_THAT(
3832 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003833 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003834
3835 // Request key-frame. The forces a dummy frame down into the encoder.
3836 fake_encoder_.ExpectNullFrame();
3837 video_stream_encoder_->SendKeyFrame();
3838 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01003839 EXPECT_THAT(
3840 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02003841 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01003842
3843 video_stream_encoder_->Stop();
3844}
Erik Språngb7cb7b52019-02-26 15:52:33 +01003845
3846TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
3847 // Configure internal source factory and setup test again.
3848 encoder_factory_.SetHasInternalSource(true);
3849 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02003850 video_stream_encoder_->OnBitrateUpdated(
3851 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01003852
3853 int64_t timestamp = 1;
3854 EncodedImage image;
3855 image.Allocate(kTargetBitrateBps / kDefaultFramerate / 8);
3856 image.capture_time_ms_ = ++timestamp;
3857 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
3858 const int64_t kEncodeFinishDelayMs = 10;
3859 image.timing_.encode_start_ms = timestamp;
3860 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
3861 fake_encoder_.InjectEncodedImage(image);
3862 // Wait for frame without incrementing clock.
3863 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3864 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
3865 // capture timestamp should be kEncodeFinishDelayMs in the past.
3866 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
3867 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
3868 kEncodeFinishDelayMs);
3869
3870 video_stream_encoder_->Stop();
3871}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02003872
3873TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
3874 // Configure internal source factory and setup test again.
3875 encoder_factory_.SetHasInternalSource(true);
3876 ResetEncoder("H264", 1, 1, 1, false);
3877
3878 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
3879 image._frameType = VideoFrameType::kVideoFrameKey;
3880
3881 CodecSpecificInfo codec_specific_info;
3882 codec_specific_info.codecType = kVideoCodecH264;
3883
3884 RTPFragmentationHeader fragmentation;
3885 fragmentation.VerifyAndAllocateFragmentationHeader(1);
3886 fragmentation.fragmentationOffset[0] = 4;
3887 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
3888
3889 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
3890 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3891
3892 EXPECT_THAT(sink_.GetLastEncodedImageData(),
3893 testing::ElementsAreArray(optimal_sps));
3894 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
3895 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
3896 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
3897 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
3898
3899 video_stream_encoder_->Stop();
3900}
3901
3902TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
3903 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
3904 0x00, 0x00, 0x03, 0x03, 0xF4,
3905 0x05, 0x03, 0xC7, 0xC0};
3906
3907 // Configure internal source factory and setup test again.
3908 encoder_factory_.SetHasInternalSource(true);
3909 ResetEncoder("H264", 1, 1, 1, false);
3910
3911 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
3912 image._frameType = VideoFrameType::kVideoFrameKey;
3913
3914 CodecSpecificInfo codec_specific_info;
3915 codec_specific_info.codecType = kVideoCodecH264;
3916
3917 RTPFragmentationHeader fragmentation;
3918 fragmentation.VerifyAndAllocateFragmentationHeader(1);
3919 fragmentation.fragmentationOffset[0] = 4;
3920 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
3921
3922 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
3923 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3924
3925 EXPECT_THAT(sink_.GetLastEncodedImageData(),
3926 testing::ElementsAreArray(optimal_sps));
3927 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
3928 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
3929 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
3930 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
3931
3932 video_stream_encoder_->Stop();
3933}
3934
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02003935TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
3936 const int kFrameWidth = 1280;
3937 const int kFrameHeight = 720;
3938 const int kTargetBitrateBps = 300000; // To low for HD resolution.
3939
3940 video_stream_encoder_->OnBitrateUpdated(
3941 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
3942 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3943
3944 // Insert a first video frame. It should be dropped because of downscale in
3945 // resolution.
3946 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3947 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
3948 frame.set_rotation(kVideoRotation_270);
3949 video_source_.IncomingCapturedFrame(frame);
3950
3951 ExpectDroppedFrame();
3952
3953 // Second frame is downscaled.
3954 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3955 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
3956 frame.set_rotation(kVideoRotation_90);
3957 video_source_.IncomingCapturedFrame(frame);
3958
3959 WaitForEncodedFrame(timestamp_ms);
3960 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
3961
3962 // Insert another frame, also downscaled.
3963 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3964 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
3965 frame.set_rotation(kVideoRotation_180);
3966 video_source_.IncomingCapturedFrame(frame);
3967
3968 WaitForEncodedFrame(timestamp_ms);
3969 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
3970
3971 video_stream_encoder_->Stop();
3972}
3973
perkj26091b12016-09-01 01:17:40 -07003974} // namespace webrtc