blob: 4f0f23374aa8259b54648e01f597eac1f3e2fe02 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080021#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010023#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020024#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020025#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010026#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020027#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070028#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080029#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020030#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010032#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010036#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020037#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "system_wrappers/include/sleep.h"
39#include "test/encoder_settings.h"
40#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020041#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "test/frame_generator.h"
43#include "test/gmock.h"
44#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020045#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070047
48namespace webrtc {
49
sprangb1ca0732017-02-01 08:38:12 -080050using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080051using ::testing::_;
kthelgason876222f2016-11-29 01:44:11 -080052
perkj803d97f2016-11-01 11:45:46 -070053namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020054const int kMinPixelsPerFrame = 320 * 180;
55const int kMinFramerateFps = 2;
56const int kMinBalancedFramerateFps = 7;
57const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080058const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010059const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020060const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010061const uint32_t kSimulcastTargetBitrateBps = 3150000;
62const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080063const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070064const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080066
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020067uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
68 0x00, 0x00, 0x03, 0x03, 0xF4,
69 0x05, 0x03, 0xC7, 0xE0, 0x1B,
70 0x41, 0x10, 0x8D, 0x00};
71
perkj803d97f2016-11-01 11:45:46 -070072class TestBuffer : public webrtc::I420Buffer {
73 public:
74 TestBuffer(rtc::Event* event, int width, int height)
75 : I420Buffer(width, height), event_(event) {}
76
77 private:
78 friend class rtc::RefCountedObject<TestBuffer>;
79 ~TestBuffer() override {
80 if (event_)
81 event_->Set();
82 }
83 rtc::Event* const event_;
84};
85
Noah Richards51db4212019-06-12 06:59:12 -070086// A fake native buffer that can't be converted to I420.
87class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
88 public:
89 FakeNativeBuffer(rtc::Event* event, int width, int height)
90 : event_(event), width_(width), height_(height) {}
91 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
92 int width() const override { return width_; }
93 int height() const override { return height_; }
94 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
95 return nullptr;
96 }
97
98 private:
99 friend class rtc::RefCountedObject<FakeNativeBuffer>;
100 ~FakeNativeBuffer() override {
101 if (event_)
102 event_->Set();
103 }
104 rtc::Event* const event_;
105 const int width_;
106 const int height_;
107};
108
Niels Möller7dc26b72017-12-06 10:27:48 +0100109class CpuOveruseDetectorProxy : public OveruseFrameDetector {
110 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200111 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
112 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100113 last_target_framerate_fps_(-1) {}
114 virtual ~CpuOveruseDetectorProxy() {}
115
116 void OnTargetFramerateUpdated(int framerate_fps) override {
117 rtc::CritScope cs(&lock_);
118 last_target_framerate_fps_ = framerate_fps;
119 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
120 }
121
122 int GetLastTargetFramerate() {
123 rtc::CritScope cs(&lock_);
124 return last_target_framerate_fps_;
125 }
126
Niels Möller4db138e2018-04-19 09:04:13 +0200127 CpuOveruseOptions GetOptions() { return options_; }
128
Niels Möller7dc26b72017-12-06 10:27:48 +0100129 private:
130 rtc::CriticalSection lock_;
131 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
132};
133
mflodmancc3d4422017-08-03 08:27:51 -0700134class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700135 public:
Niels Möller213618e2018-07-24 09:29:58 +0200136 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200137 const VideoStreamEncoderSettings& settings,
138 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100139 : VideoStreamEncoder(Clock::GetRealTimeClock(),
140 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200141 stats_proxy,
142 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200143 std::unique_ptr<OveruseFrameDetector>(
144 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100145 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200146 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700147
sprangb1ca0732017-02-01 08:38:12 -0800148 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100149 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800150 encoder_queue()->PostTask([this, &event, reason, down] {
Åsa Perssonf5e5d252019-08-16 17:24:59 +0200151 if (down)
152 AdaptDown(reason);
153 else
154 AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700155 event.Set();
156 });
perkj070ba852017-02-16 15:46:27 -0800157 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700158 }
159
kthelgason2fc52542017-03-03 00:24:41 -0800160 // This is used as a synchronisation mechanism, to make sure that the
161 // encoder queue is not blocked before we start sending it frames.
162 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100163 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200164 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800165 ASSERT_TRUE(event.Wait(5000));
166 }
167
sprangb1ca0732017-02-01 08:38:12 -0800168 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800169
sprangb1ca0732017-02-01 08:38:12 -0800170 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800171
sprangb1ca0732017-02-01 08:38:12 -0800172 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800173
sprangb1ca0732017-02-01 08:38:12 -0800174 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700175
Niels Möller7dc26b72017-12-06 10:27:48 +0100176 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700177};
178
asapersson5f7226f2016-11-25 04:37:00 -0800179class VideoStreamFactory
180 : public VideoEncoderConfig::VideoStreamFactoryInterface {
181 public:
sprangfda496a2017-06-15 04:21:07 -0700182 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
183 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800184 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700185 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800186 }
187
188 private:
189 std::vector<VideoStream> CreateEncoderStreams(
190 int width,
191 int height,
192 const VideoEncoderConfig& encoder_config) override {
193 std::vector<VideoStream> streams =
194 test::CreateVideoStreams(width, height, encoder_config);
195 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100196 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700197 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800198 }
199 return streams;
200 }
sprangfda496a2017-06-15 04:21:07 -0700201
asapersson5f7226f2016-11-25 04:37:00 -0800202 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700203 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800204};
205
Noah Richards51db4212019-06-12 06:59:12 -0700206// Simulates simulcast behavior and makes highest stream resolutions divisible
207// by 4.
208class CroppingVideoStreamFactory
209 : public VideoEncoderConfig::VideoStreamFactoryInterface {
210 public:
211 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
212 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
213 EXPECT_GT(num_temporal_layers, 0u);
214 EXPECT_GT(framerate, 0);
215 }
216
217 private:
218 std::vector<VideoStream> CreateEncoderStreams(
219 int width,
220 int height,
221 const VideoEncoderConfig& encoder_config) override {
222 std::vector<VideoStream> streams = test::CreateVideoStreams(
223 width - width % 4, height - height % 4, encoder_config);
224 for (VideoStream& stream : streams) {
225 stream.num_temporal_layers = num_temporal_layers_;
226 stream.max_framerate = framerate_;
227 }
228 return streams;
229 }
230
231 const size_t num_temporal_layers_;
232 const int framerate_;
233};
234
sprangb1ca0732017-02-01 08:38:12 -0800235class AdaptingFrameForwarder : public test::FrameForwarder {
236 public:
237 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700238 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800239
240 void set_adaptation_enabled(bool enabled) {
241 rtc::CritScope cs(&crit_);
242 adaptation_enabled_ = enabled;
243 }
244
asaperssonfab67072017-04-04 05:51:49 -0700245 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800246 rtc::CritScope cs(&crit_);
247 return adaptation_enabled_;
248 }
249
asapersson09f05612017-05-15 23:40:18 -0700250 rtc::VideoSinkWants last_wants() const {
251 rtc::CritScope cs(&crit_);
252 return last_wants_;
253 }
254
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200255 absl::optional<int> last_sent_width() const { return last_width_; }
256 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800257
sprangb1ca0732017-02-01 08:38:12 -0800258 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
259 int cropped_width = 0;
260 int cropped_height = 0;
261 int out_width = 0;
262 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700263 if (adaption_enabled()) {
264 if (adapter_.AdaptFrameResolution(
265 video_frame.width(), video_frame.height(),
266 video_frame.timestamp_us() * 1000, &cropped_width,
267 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100268 VideoFrame adapted_frame =
269 VideoFrame::Builder()
270 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
271 nullptr, out_width, out_height))
272 .set_timestamp_rtp(99)
273 .set_timestamp_ms(99)
274 .set_rotation(kVideoRotation_0)
275 .build();
sprangc5d62e22017-04-02 23:53:04 -0700276 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
277 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800278 last_width_.emplace(adapted_frame.width());
279 last_height_.emplace(adapted_frame.height());
280 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200281 last_width_ = absl::nullopt;
282 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700283 }
sprangb1ca0732017-02-01 08:38:12 -0800284 } else {
285 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800286 last_width_.emplace(video_frame.width());
287 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800288 }
289 }
290
291 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
292 const rtc::VideoSinkWants& wants) override {
293 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700294 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700295 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
296 wants.max_pixel_count,
297 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800298 test::FrameForwarder::AddOrUpdateSink(sink, wants);
299 }
sprangb1ca0732017-02-01 08:38:12 -0800300 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700301 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
302 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200303 absl::optional<int> last_width_;
304 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800305};
sprangc5d62e22017-04-02 23:53:04 -0700306
Niels Möller213618e2018-07-24 09:29:58 +0200307// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700308class MockableSendStatisticsProxy : public SendStatisticsProxy {
309 public:
310 MockableSendStatisticsProxy(Clock* clock,
311 const VideoSendStream::Config& config,
312 VideoEncoderConfig::ContentType content_type)
313 : SendStatisticsProxy(clock, config, content_type) {}
314
315 VideoSendStream::Stats GetStats() override {
316 rtc::CritScope cs(&lock_);
317 if (mock_stats_)
318 return *mock_stats_;
319 return SendStatisticsProxy::GetStats();
320 }
321
Niels Möller213618e2018-07-24 09:29:58 +0200322 int GetInputFrameRate() const override {
323 rtc::CritScope cs(&lock_);
324 if (mock_stats_)
325 return mock_stats_->input_frame_rate;
326 return SendStatisticsProxy::GetInputFrameRate();
327 }
sprangc5d62e22017-04-02 23:53:04 -0700328 void SetMockStats(const VideoSendStream::Stats& stats) {
329 rtc::CritScope cs(&lock_);
330 mock_stats_.emplace(stats);
331 }
332
333 void ResetMockStats() {
334 rtc::CritScope cs(&lock_);
335 mock_stats_.reset();
336 }
337
338 private:
339 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200340 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700341};
342
sprang4847ae62017-06-27 07:06:52 -0700343class MockBitrateObserver : public VideoBitrateAllocationObserver {
344 public:
Erik Språng566124a2018-04-23 12:32:22 +0200345 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700346};
347
perkj803d97f2016-11-01 11:45:46 -0700348} // namespace
349
mflodmancc3d4422017-08-03 08:27:51 -0700350class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700351 public:
352 static const int kDefaultTimeoutMs = 30 * 1000;
353
mflodmancc3d4422017-08-03 08:27:51 -0700354 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700355 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700356 codec_width_(320),
357 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200358 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200359 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700360 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200361 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700362 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700363 Clock::GetRealTimeClock(),
364 video_send_config_,
365 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700366 sink_(&fake_encoder_) {}
367
368 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700369 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700370 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200371 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800372 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200373 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200374 video_send_config_.rtp.payload_name = "FAKE";
375 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700376
Per512ecb32016-09-23 15:52:06 +0200377 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200378 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700379 video_encoder_config.video_stream_factory =
380 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100381 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700382
383 // Framerate limit is specified by the VideoStreamFactory.
384 std::vector<VideoStream> streams =
385 video_encoder_config.video_stream_factory->CreateEncoderStreams(
386 codec_width_, codec_height_, video_encoder_config);
387 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200388 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700389
Niels Möllerf1338562018-04-26 09:51:47 +0200390 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800391 }
392
Niels Möllerf1338562018-04-26 09:51:47 +0200393 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700394 if (video_stream_encoder_)
395 video_stream_encoder_->Stop();
396 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200397 stats_proxy_.get(), video_send_config_.encoder_settings,
398 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700399 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
400 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700401 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700402 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
403 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200404 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700405 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800406 }
407
408 void ResetEncoder(const std::string& payload_name,
409 size_t num_streams,
410 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700411 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700412 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200413 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800414
415 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200416 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800417 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100418 video_encoder_config.max_bitrate_bps =
419 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800420 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700421 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
422 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700423 video_encoder_config.content_type =
424 screenshare ? VideoEncoderConfig::ContentType::kScreen
425 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700426 if (payload_name == "VP9") {
427 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
428 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
429 video_encoder_config.encoder_specific_settings =
430 new rtc::RefCountedObject<
431 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
432 }
Niels Möllerf1338562018-04-26 09:51:47 +0200433 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700434 }
435
sprang57c2fff2017-01-16 06:24:02 -0800436 VideoFrame CreateFrame(int64_t ntp_time_ms,
437 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100438 VideoFrame frame =
439 VideoFrame::Builder()
440 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
441 destruction_event, codec_width_, codec_height_))
442 .set_timestamp_rtp(99)
443 .set_timestamp_ms(99)
444 .set_rotation(kVideoRotation_0)
445 .build();
sprang57c2fff2017-01-16 06:24:02 -0800446 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700447 return frame;
448 }
449
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100450 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
451 rtc::Event* destruction_event,
452 int offset_x) const {
453 VideoFrame frame =
454 VideoFrame::Builder()
455 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
456 destruction_event, codec_width_, codec_height_))
457 .set_timestamp_rtp(99)
458 .set_timestamp_ms(99)
459 .set_rotation(kVideoRotation_0)
460 .set_update_rect({offset_x, 0, 1, 1})
461 .build();
462 frame.set_ntp_time_ms(ntp_time_ms);
463 return frame;
464 }
465
sprang57c2fff2017-01-16 06:24:02 -0800466 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100467 VideoFrame frame =
468 VideoFrame::Builder()
469 .set_video_frame_buffer(
470 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
471 .set_timestamp_rtp(99)
472 .set_timestamp_ms(99)
473 .set_rotation(kVideoRotation_0)
474 .build();
sprang57c2fff2017-01-16 06:24:02 -0800475 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700476 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700477 return frame;
478 }
479
Noah Richards51db4212019-06-12 06:59:12 -0700480 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
481 rtc::Event* destruction_event,
482 int width,
483 int height) const {
484 VideoFrame frame =
485 VideoFrame::Builder()
486 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
487 destruction_event, width, height))
488 .set_timestamp_rtp(99)
489 .set_timestamp_ms(99)
490 .set_rotation(kVideoRotation_0)
491 .build();
492 frame.set_ntp_time_ms(ntp_time_ms);
493 return frame;
494 }
495
496 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
497 rtc::Event* destruction_event) const {
498 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
499 codec_height_);
500 }
501
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100502 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
503 MockBitrateObserver bitrate_observer;
504 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
505
506 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
507 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200508 video_stream_encoder_->OnBitrateUpdated(
509 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
510 DataRate::bps(kTargetBitrateBps), 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100511
512 video_source_.IncomingCapturedFrame(
513 CreateFrame(1, codec_width_, codec_height_));
514 WaitForEncodedFrame(1);
515 }
516
asapersson02465b82017-04-10 01:12:52 -0700517 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700518 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700519 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
520 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700521 }
522
asapersson09f05612017-05-15 23:40:18 -0700523 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
524 const rtc::VideoSinkWants& wants2) {
525 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
526 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
527 }
528
Åsa Persson8c1bf952018-09-13 10:42:19 +0200529 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
530 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
531 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
532 EXPECT_FALSE(wants.target_pixel_count);
533 }
534
asapersson09f05612017-05-15 23:40:18 -0700535 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
536 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200537 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700538 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
539 EXPECT_GT(wants1.max_pixel_count, 0);
540 }
541
542 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
543 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200544 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700545 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
546 }
547
asaperssonf7e294d2017-06-13 23:25:22 -0700548 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
549 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200550 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700551 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
552 }
553
554 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
555 const rtc::VideoSinkWants& wants2) {
556 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
557 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
558 }
559
560 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
561 const rtc::VideoSinkWants& wants2) {
562 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
563 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
564 }
565
566 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
567 const rtc::VideoSinkWants& wants2) {
568 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
569 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
570 EXPECT_GT(wants1.max_pixel_count, 0);
571 }
572
573 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
574 const rtc::VideoSinkWants& wants2) {
575 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
576 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
577 }
578
asapersson09f05612017-05-15 23:40:18 -0700579 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
580 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200581 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700582 EXPECT_LT(wants.max_pixel_count, pixel_count);
583 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700584 }
585
586 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
587 EXPECT_LT(wants.max_framerate_fps, fps);
588 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
589 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700590 }
591
asaperssonf7e294d2017-06-13 23:25:22 -0700592 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
593 int expected_fps) {
594 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
595 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
596 EXPECT_FALSE(wants.target_pixel_count);
597 }
598
Jonathan Yubc771b72017-12-08 17:04:29 -0800599 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
600 int last_frame_pixels) {
601 // Balanced mode should always scale FPS to the desired range before
602 // attempting to scale resolution.
603 int fps_limit = wants.max_framerate_fps;
604 if (last_frame_pixels <= 320 * 240) {
605 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
606 } else if (last_frame_pixels <= 480 * 270) {
607 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
608 } else if (last_frame_pixels <= 640 * 480) {
609 EXPECT_LE(15, fps_limit);
610 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200611 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800612 }
613 }
614
sprang4847ae62017-06-27 07:06:52 -0700615 void WaitForEncodedFrame(int64_t expected_ntp_time) {
616 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200617 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700618 }
619
620 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
621 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200622 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700623 return ok;
624 }
625
626 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
627 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200628 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700629 }
630
631 void ExpectDroppedFrame() {
632 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200633 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700634 }
635
636 bool WaitForFrame(int64_t timeout_ms) {
637 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200638 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700639 return ok;
640 }
641
perkj26091b12016-09-01 01:17:40 -0700642 class TestEncoder : public test::FakeEncoder {
643 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100644 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700645
asaperssonfab67072017-04-04 05:51:49 -0700646 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800647 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700648 return config_;
649 }
650
651 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800652 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700653 block_next_encode_ = true;
654 }
655
Erik Språngaed30702018-11-05 12:57:17 +0100656 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800657 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100658 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100659 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100660 if (quality_scaling_) {
661 info.scaling_settings =
662 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
663 }
664 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100665 for (int i = 0; i < kMaxSpatialLayers; ++i) {
666 if (temporal_layers_supported_[i]) {
667 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
668 info.fps_allocation[i].resize(num_layers);
669 }
670 }
Erik Språngaed30702018-11-05 12:57:17 +0100671 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200672
673 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Erik Språngaed30702018-11-05 12:57:17 +0100674 return info;
kthelgason876222f2016-11-29 01:44:11 -0800675 }
676
Erik Språngb7cb7b52019-02-26 15:52:33 +0100677 int32_t RegisterEncodeCompleteCallback(
678 EncodedImageCallback* callback) override {
679 rtc::CritScope lock(&local_crit_sect_);
680 encoded_image_callback_ = callback;
681 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
682 }
683
perkjfa10b552016-10-02 23:45:26 -0700684 void ContinueEncode() { continue_encode_event_.Set(); }
685
686 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
687 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800688 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700689 EXPECT_EQ(timestamp_, timestamp);
690 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
691 }
692
kthelgason2fc52542017-03-03 00:24:41 -0800693 void SetQualityScaling(bool b) {
694 rtc::CritScope lock(&local_crit_sect_);
695 quality_scaling_ = b;
696 }
kthelgasonad9010c2017-02-14 00:46:51 -0800697
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100698 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
699 rtc::CritScope lock(&local_crit_sect_);
700 is_hardware_accelerated_ = is_hardware_accelerated;
701 }
702
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100703 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
704 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
705 rtc::CritScope lock(&local_crit_sect_);
706 temporal_layers_supported_[spatial_idx] = supported;
707 }
708
Sergey Silkin6456e352019-07-08 17:56:40 +0200709 void SetResolutionBitrateLimits(
710 std::vector<ResolutionBitrateLimits> thresholds) {
711 rtc::CritScope cs(&local_crit_sect_);
712 resolution_bitrate_limits_ = thresholds;
713 }
714
sprangfe627f32017-03-29 08:24:59 -0700715 void ForceInitEncodeFailure(bool force_failure) {
716 rtc::CritScope lock(&local_crit_sect_);
717 force_init_encode_failed_ = force_failure;
718 }
719
Niels Möller6bb5ab92019-01-11 11:11:10 +0100720 void SimulateOvershoot(double rate_factor) {
721 rtc::CritScope lock(&local_crit_sect_);
722 rate_factor_ = rate_factor;
723 }
724
Erik Språngd7329ca2019-02-21 21:19:53 +0100725 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100726 rtc::CritScope lock(&local_crit_sect_);
727 return last_framerate_;
728 }
729
Erik Språngd7329ca2019-02-21 21:19:53 +0100730 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100731 rtc::CritScope lock(&local_crit_sect_);
732 return last_update_rect_;
733 }
734
Niels Möller87e2d782019-03-07 10:18:23 +0100735 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100736 rtc::CritScope lock(&local_crit_sect_);
737 return last_frame_types_;
738 }
739
740 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100741 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100742 keyframe ? VideoFrameType::kVideoFrameKey
743 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100744 {
745 rtc::CritScope lock(&local_crit_sect_);
746 last_frame_types_ = frame_type;
747 }
Niels Möllerb859b322019-03-07 12:40:01 +0100748 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100749 }
750
Erik Språngb7cb7b52019-02-26 15:52:33 +0100751 void InjectEncodedImage(const EncodedImage& image) {
752 rtc::CritScope lock(&local_crit_sect_);
753 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
754 }
755
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200756 void InjectEncodedImage(const EncodedImage& image,
757 const CodecSpecificInfo* codec_specific_info,
758 const RTPFragmentationHeader* fragmentation) {
759 rtc::CritScope lock(&local_crit_sect_);
760 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
761 fragmentation);
762 }
763
Erik Språngd7329ca2019-02-21 21:19:53 +0100764 void ExpectNullFrame() {
765 rtc::CritScope lock(&local_crit_sect_);
766 expect_null_frame_ = true;
767 }
768
Erik Språng5056af02019-09-02 15:53:11 +0200769 absl::optional<VideoEncoder::RateControlParameters>
770 GetAndResetLastRateControlSettings() {
771 auto settings = last_rate_control_settings_;
772 last_rate_control_settings_.reset();
773 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100774 }
775
Sergey Silkin5ee69672019-07-02 14:18:34 +0200776 int GetNumEncoderInitializations() const {
777 rtc::CritScope lock(&local_crit_sect_);
778 return num_encoder_initializations_;
779 }
780
perkjfa10b552016-10-02 23:45:26 -0700781 private:
perkj26091b12016-09-01 01:17:40 -0700782 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100783 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700784 bool block_encode;
785 {
brandtre78d2662017-01-16 05:57:16 -0800786 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100787 if (expect_null_frame_) {
788 EXPECT_EQ(input_image.timestamp(), 0u);
789 EXPECT_EQ(input_image.width(), 1);
790 last_frame_types_ = *frame_types;
791 expect_null_frame_ = false;
792 } else {
793 EXPECT_GT(input_image.timestamp(), timestamp_);
794 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
795 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
796 }
perkj26091b12016-09-01 01:17:40 -0700797
798 timestamp_ = input_image.timestamp();
799 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700800 last_input_width_ = input_image.width();
801 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700802 block_encode = block_next_encode_;
803 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100804 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100805 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700806 }
Niels Möllerb859b322019-03-07 12:40:01 +0100807 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700808 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700809 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700810 return result;
811 }
812
sprangfe627f32017-03-29 08:24:59 -0700813 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200814 const Settings& settings) override {
815 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200816
sprangfe627f32017-03-29 08:24:59 -0700817 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100818 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200819
820 ++num_encoder_initializations_;
821
Erik Språng82fad3d2018-03-21 09:57:23 +0100822 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700823 // Simulate setting up temporal layers, in order to validate the life
824 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100825 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200826 frame_buffer_controller_ =
827 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700828 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100829 if (force_init_encode_failed_) {
830 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700831 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100832 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100833
Erik Språngb7cb7b52019-02-26 15:52:33 +0100834 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700835 return res;
836 }
837
Erik Språngb7cb7b52019-02-26 15:52:33 +0100838 int32_t Release() override {
839 rtc::CritScope lock(&local_crit_sect_);
840 EXPECT_NE(initialized_, EncoderState::kUninitialized);
841 initialized_ = EncoderState::kUninitialized;
842 return FakeEncoder::Release();
843 }
844
Erik Språng16cb8f52019-04-12 13:59:09 +0200845 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100846 rtc::CritScope lock(&local_crit_sect_);
847 VideoBitrateAllocation adjusted_rate_allocation;
848 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
849 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200850 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100851 adjusted_rate_allocation.SetBitrate(
852 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200853 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100854 rate_factor_));
855 }
856 }
857 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200858 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200859 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200860 RateControlParameters adjusted_paramters = parameters;
861 adjusted_paramters.bitrate = adjusted_rate_allocation;
862 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100863 }
864
brandtre78d2662017-01-16 05:57:16 -0800865 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100866 enum class EncoderState {
867 kUninitialized,
868 kInitializationFailed,
869 kInitialized
870 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
871 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700872 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700873 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700874 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
875 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
876 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
877 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
878 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100879 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100880 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700881 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100882 absl::optional<bool>
883 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
884 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700885 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100886 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
887 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200888 absl::optional<VideoEncoder::RateControlParameters>
889 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100890 VideoFrame::UpdateRect last_update_rect_
891 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100892 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100893 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100894 EncodedImageCallback* encoded_image_callback_
895 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 11:20:09 +0200896 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200897 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +0200898 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
899 RTC_GUARDED_BY(local_crit_sect_);
perkj26091b12016-09-01 01:17:40 -0700900 };
901
mflodmancc3d4422017-08-03 08:27:51 -0700902 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700903 public:
904 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100905 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700906
perkj26091b12016-09-01 01:17:40 -0700907 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700908 EXPECT_TRUE(
909 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
910 }
911
912 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
913 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700914 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700915 if (!encoded_frame_event_.Wait(timeout_ms))
916 return false;
perkj26091b12016-09-01 01:17:40 -0700917 {
918 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800919 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700920 }
921 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700922 return true;
perkj26091b12016-09-01 01:17:40 -0700923 }
924
sprangb1ca0732017-02-01 08:38:12 -0800925 void WaitForEncodedFrame(uint32_t expected_width,
926 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700927 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100928 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700929 }
930
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100931 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700932 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800933 uint32_t width = 0;
934 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800935 {
936 rtc::CritScope lock(&crit_);
937 width = last_width_;
938 height = last_height_;
939 }
940 EXPECT_EQ(expected_height, height);
941 EXPECT_EQ(expected_width, width);
942 }
943
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200944 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
945 VideoRotation rotation;
946 {
947 rtc::CritScope lock(&crit_);
948 rotation = last_rotation_;
949 }
950 EXPECT_EQ(expected_rotation, rotation);
951 }
952
kthelgason2fc52542017-03-03 00:24:41 -0800953 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800954
sprangc5d62e22017-04-02 23:53:04 -0700955 bool WaitForFrame(int64_t timeout_ms) {
956 return encoded_frame_event_.Wait(timeout_ms);
957 }
958
perkj26091b12016-09-01 01:17:40 -0700959 void SetExpectNoFrames() {
960 rtc::CritScope lock(&crit_);
961 expect_frames_ = false;
962 }
963
asaperssonfab67072017-04-04 05:51:49 -0700964 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200965 rtc::CritScope lock(&crit_);
966 return number_of_reconfigurations_;
967 }
968
asaperssonfab67072017-04-04 05:51:49 -0700969 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200970 rtc::CritScope lock(&crit_);
971 return min_transmit_bitrate_bps_;
972 }
973
Erik Språngd7329ca2019-02-21 21:19:53 +0100974 void SetNumExpectedLayers(size_t num_layers) {
975 rtc::CritScope lock(&crit_);
976 num_expected_layers_ = num_layers;
977 }
978
Erik Språngb7cb7b52019-02-26 15:52:33 +0100979 int64_t GetLastCaptureTimeMs() const {
980 rtc::CritScope lock(&crit_);
981 return last_capture_time_ms_;
982 }
983
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200984 std::vector<uint8_t> GetLastEncodedImageData() {
985 rtc::CritScope lock(&crit_);
986 return std::move(last_encoded_image_data_);
987 }
988
989 RTPFragmentationHeader GetLastFragmentation() {
990 rtc::CritScope lock(&crit_);
991 return std::move(last_fragmentation_);
992 }
993
perkj26091b12016-09-01 01:17:40 -0700994 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700995 Result OnEncodedImage(
996 const EncodedImage& encoded_image,
997 const CodecSpecificInfo* codec_specific_info,
998 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200999 rtc::CritScope lock(&crit_);
1000 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001001 last_encoded_image_data_ = std::vector<uint8_t>(
1002 encoded_image.data(), encoded_image.data() + encoded_image.size());
1003 if (fragmentation) {
1004 last_fragmentation_.CopyFrom(*fragmentation);
1005 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001006 uint32_t timestamp = encoded_image.Timestamp();
1007 if (last_timestamp_ != timestamp) {
1008 num_received_layers_ = 1;
1009 } else {
1010 ++num_received_layers_;
1011 }
1012 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001013 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001014 last_width_ = encoded_image._encodedWidth;
1015 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001016 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001017 if (num_received_layers_ == num_expected_layers_) {
1018 encoded_frame_event_.Set();
1019 }
sprangb1ca0732017-02-01 08:38:12 -08001020 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001021 }
1022
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001023 void OnEncoderConfigurationChanged(
1024 std::vector<VideoStream> streams,
1025 VideoEncoderConfig::ContentType content_type,
1026 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001027 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001028 ++number_of_reconfigurations_;
1029 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1030 }
1031
perkj26091b12016-09-01 01:17:40 -07001032 rtc::CriticalSection crit_;
1033 TestEncoder* test_encoder_;
1034 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001035 std::vector<uint8_t> last_encoded_image_data_;
1036 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001037 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001038 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001039 uint32_t last_height_ = 0;
1040 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001041 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001042 size_t num_expected_layers_ = 1;
1043 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001044 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001045 int number_of_reconfigurations_ = 0;
1046 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001047 };
1048
Sergey Silkin5ee69672019-07-02 14:18:34 +02001049 class VideoBitrateAllocatorProxyFactory
1050 : public VideoBitrateAllocatorFactory {
1051 public:
1052 VideoBitrateAllocatorProxyFactory()
1053 : bitrate_allocator_factory_(
1054 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1055
1056 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1057 const VideoCodec& codec) override {
1058 rtc::CritScope lock(&crit_);
1059 codec_config_ = codec;
1060 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1061 }
1062
1063 VideoCodec codec_config() const {
1064 rtc::CritScope lock(&crit_);
1065 return codec_config_;
1066 }
1067
1068 private:
1069 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1070
1071 rtc::CriticalSection crit_;
1072 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1073 };
1074
perkj26091b12016-09-01 01:17:40 -07001075 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001076 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001077 int codec_width_;
1078 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001079 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001080 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001081 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001082 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001083 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001084 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001085 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001086 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001087 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001088 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001089};
1090
mflodmancc3d4422017-08-03 08:27:51 -07001091TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001092 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001093 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1094 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001095 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001096 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001097 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001098 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001099 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001100}
1101
mflodmancc3d4422017-08-03 08:27:51 -07001102TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001103 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001104 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001105 // The encoder will cache up to one frame for a short duration. Adding two
1106 // frames means that the first frame will be dropped and the second frame will
1107 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001108 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001109 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001110 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001111
Erik Språng4c6ca302019-04-08 15:14:01 +02001112 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001113 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1114 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001115
Sebastian Janssona3177052018-04-10 13:05:49 +02001116 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001117 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001118 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1119
1120 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001121 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001122}
1123
mflodmancc3d4422017-08-03 08:27:51 -07001124TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001125 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001126 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1127 DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001128 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001129 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001130
Florent Castellia8336d32019-09-09 13:36:55 +02001131 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
1132 DataRate::bps(0), 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001133 // The encoder will cache up to one frame for a short duration. Adding two
1134 // frames means that the first frame will be dropped and the second frame will
1135 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001136 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001137 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001138
Erik Språng4c6ca302019-04-08 15:14:01 +02001139 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001140 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1141 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001142 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001143 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1144 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001145 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001146}
1147
mflodmancc3d4422017-08-03 08:27:51 -07001148TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001149 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001150 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1151 DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001152 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001153 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001154
1155 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001156 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001157
perkja49cbd32016-09-16 07:53:41 -07001158 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001159 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001160 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001161}
1162
mflodmancc3d4422017-08-03 08:27:51 -07001163TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001164 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001165 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1166 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001167
perkja49cbd32016-09-16 07:53:41 -07001168 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001169 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001170
mflodmancc3d4422017-08-03 08:27:51 -07001171 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001172 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001173 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001174 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1175 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001176}
1177
mflodmancc3d4422017-08-03 08:27:51 -07001178TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001179 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001180 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1181 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001182
1183 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001184 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001185 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001186 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1187 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001188 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1189 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001190 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001191 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001192
mflodmancc3d4422017-08-03 08:27:51 -07001193 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001194}
1195
Noah Richards51db4212019-06-12 06:59:12 -07001196TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1197 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001198 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1199 DataRate::bps(kTargetBitrateBps), 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001200
1201 rtc::Event frame_destroyed_event;
1202 video_source_.IncomingCapturedFrame(
1203 CreateFakeNativeFrame(1, &frame_destroyed_event));
1204 ExpectDroppedFrame();
1205 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1206 video_stream_encoder_->Stop();
1207}
1208
1209TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1210 // Use the cropping factory.
1211 video_encoder_config_.video_stream_factory =
1212 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1213 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1214 kMaxPayloadLength);
1215 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1216
1217 // Capture a frame at codec_width_/codec_height_.
1218 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001219 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1220 DataRate::bps(kTargetBitrateBps), 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001221 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1222 WaitForEncodedFrame(1);
1223 // The encoder will have been configured once.
1224 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1225 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1226 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1227
1228 // Now send in a fake frame that needs to be cropped as the width/height
1229 // aren't divisible by 4 (see CreateEncoderStreams above).
1230 rtc::Event frame_destroyed_event;
1231 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1232 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1233 ExpectDroppedFrame();
1234 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1235 video_stream_encoder_->Stop();
1236}
1237
mflodmancc3d4422017-08-03 08:27:51 -07001238TEST_F(VideoStreamEncoderTest,
1239 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001240 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001241 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1242 DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001243 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001244
1245 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001246 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001247 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001248 // The encoder will have been configured once when the first frame is
1249 // received.
1250 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001251
1252 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001253 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001254 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001255 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001256 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001257
1258 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001259 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001260 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001261 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001262 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001263
mflodmancc3d4422017-08-03 08:27:51 -07001264 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001265}
1266
mflodmancc3d4422017-08-03 08:27:51 -07001267TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001268 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001269 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1270 DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001271
1272 // Capture a frame and wait for it to synchronize with the encoder thread.
1273 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001274 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001275 // The encoder will have been configured once.
1276 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001277 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1278 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1279
1280 codec_width_ *= 2;
1281 codec_height_ *= 2;
1282 // Capture a frame with a higher resolution and wait for it to synchronize
1283 // with the encoder thread.
1284 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001285 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001286 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1287 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001288 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001289
mflodmancc3d4422017-08-03 08:27:51 -07001290 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001291}
1292
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001293TEST_F(VideoStreamEncoderTest,
1294 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1295 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001296 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1297 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001298
1299 // Capture a frame and wait for it to synchronize with the encoder thread.
1300 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1301 WaitForEncodedFrame(1);
1302
1303 VideoEncoderConfig video_encoder_config;
1304 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1305 // Changing the max payload data length recreates encoder.
1306 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1307 kMaxPayloadLength / 2);
1308
1309 // Capture a frame and wait for it to synchronize with the encoder thread.
1310 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1311 WaitForEncodedFrame(2);
1312 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1313
1314 video_stream_encoder_->Stop();
1315}
1316
Sergey Silkin5ee69672019-07-02 14:18:34 +02001317TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1318 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001319 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1320 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001321
1322 VideoEncoderConfig video_encoder_config;
1323 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1324 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1325 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1326 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1327 kMaxPayloadLength);
1328
1329 // Capture a frame and wait for it to synchronize with the encoder thread.
1330 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1331 WaitForEncodedFrame(1);
1332 // The encoder will have been configured once when the first frame is
1333 // received.
1334 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1335 EXPECT_EQ(kTargetBitrateBps,
1336 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1337 EXPECT_EQ(kStartBitrateBps,
1338 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1339
Sergey Silkin6456e352019-07-08 17:56:40 +02001340 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1341 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001342 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1343 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1344 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1345 kMaxPayloadLength);
1346
1347 // Capture a frame and wait for it to synchronize with the encoder thread.
1348 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1349 WaitForEncodedFrame(2);
1350 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1351 // Bitrate limits have changed - rate allocator should be reconfigured,
1352 // encoder should not be reconfigured.
1353 EXPECT_EQ(kTargetBitrateBps * 2,
1354 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1355 EXPECT_EQ(kStartBitrateBps * 2,
1356 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1357 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1358
1359 video_stream_encoder_->Stop();
1360}
1361
Sergey Silkin6456e352019-07-08 17:56:40 +02001362TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001363 EncoderRecommendedBitrateLimitsDoNotOverrideAppBitrateLimits) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001364 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001365 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1366 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001367
Sergey Silkin6456e352019-07-08 17:56:40 +02001368 VideoEncoderConfig video_encoder_config;
1369 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1370 video_encoder_config.max_bitrate_bps = 0;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001371 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001372 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1373 kMaxPayloadLength);
1374
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001375 video_source_.IncomingCapturedFrame(CreateFrame(1, 360, 180));
Sergey Silkin6456e352019-07-08 17:56:40 +02001376 WaitForEncodedFrame(1);
Sergey Silkin6456e352019-07-08 17:56:40 +02001377
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001378 // Get the default bitrate limits and use them as baseline for custom
1379 // application and encoder recommended limits.
1380 const uint32_t kDefaultMinBitrateKbps =
1381 bitrate_allocator_factory_.codec_config().minBitrate;
1382 const uint32_t kDefaultMaxBitrateKbps =
1383 bitrate_allocator_factory_.codec_config().maxBitrate;
1384 const uint32_t kEncMinBitrateKbps = kDefaultMinBitrateKbps * 2;
1385 const uint32_t kEncMaxBitrateKbps = kDefaultMaxBitrateKbps * 2;
1386 const uint32_t kAppMinBitrateKbps = kDefaultMinBitrateKbps * 3;
1387 const uint32_t kAppMaxBitrateKbps = kDefaultMaxBitrateKbps * 3;
1388
1389 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1390 codec_width_ * codec_height_, kEncMinBitrateKbps * 1000,
1391 kEncMinBitrateKbps * 1000, kEncMaxBitrateKbps * 1000);
1392 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1393
1394 // Change resolution. This will trigger encoder re-configuration and video
1395 // stream encoder will pick up the bitrate limits recommended by encoder.
1396 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1397 WaitForEncodedFrame(2);
1398 video_source_.IncomingCapturedFrame(CreateFrame(3, 360, 180));
1399 WaitForEncodedFrame(3);
1400
1401 // App bitrate limits are not set - bitrate limits recommended by encoder
1402 // should be used.
1403 EXPECT_EQ(kEncMaxBitrateKbps,
1404 bitrate_allocator_factory_.codec_config().maxBitrate);
1405 EXPECT_EQ(kEncMinBitrateKbps,
1406 bitrate_allocator_factory_.codec_config().minBitrate);
1407
1408 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1409 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
1410 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1411 kMaxPayloadLength);
1412 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1413 WaitForEncodedFrame(4);
1414
1415 // App limited the max bitrate - bitrate limits recommended by encoder should
1416 // not be applied.
1417 EXPECT_EQ(kAppMaxBitrateKbps,
1418 bitrate_allocator_factory_.codec_config().maxBitrate);
1419 EXPECT_EQ(kDefaultMinBitrateKbps,
1420 bitrate_allocator_factory_.codec_config().minBitrate);
1421
1422 video_encoder_config.max_bitrate_bps = 0;
1423 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1424 kAppMinBitrateKbps * 1000;
1425 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1426 kMaxPayloadLength);
1427 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1428 WaitForEncodedFrame(5);
1429
1430 // App limited the min bitrate - bitrate limits recommended by encoder should
1431 // not be applied.
1432 EXPECT_EQ(kDefaultMaxBitrateKbps,
1433 bitrate_allocator_factory_.codec_config().maxBitrate);
1434 EXPECT_EQ(kAppMinBitrateKbps,
1435 bitrate_allocator_factory_.codec_config().minBitrate);
1436
1437 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1438 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1439 kAppMinBitrateKbps * 1000;
Sergey Silkin6456e352019-07-08 17:56:40 +02001440 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1441 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001442 video_source_.IncomingCapturedFrame(CreateFrame(6, nullptr));
1443 WaitForEncodedFrame(6);
Sergey Silkin6456e352019-07-08 17:56:40 +02001444
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001445 // App limited both min and max bitrates - bitrate limits recommended by
1446 // encoder should not be applied.
1447 EXPECT_EQ(kAppMaxBitrateKbps,
1448 bitrate_allocator_factory_.codec_config().maxBitrate);
1449 EXPECT_EQ(kAppMinBitrateKbps,
1450 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001451
1452 video_stream_encoder_->Stop();
1453}
1454
1455TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001456 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001457 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001458 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1459 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001460
1461 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001462 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001463 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001464 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001465 fake_encoder_.SetResolutionBitrateLimits(
1466 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1467
1468 VideoEncoderConfig video_encoder_config;
1469 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1470 video_encoder_config.max_bitrate_bps = 0;
1471 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1472 kMaxPayloadLength);
1473
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001474 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001475 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1476 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001477 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1478 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001479 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1480 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1481
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001482 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001483 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1484 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001485 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1486 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001487 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1488 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1489
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001490 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001491 // encoder for 360p should be used.
1492 video_source_.IncomingCapturedFrame(
1493 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1494 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001495 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1496 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001497 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1498 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1499
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001500 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001501 // ignored.
1502 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1503 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001504 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1505 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001506 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1507 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001508 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1509 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001510 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1511 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1512
1513 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1514 // for 270p should be used.
1515 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1516 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001517 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1518 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001519 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1520 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1521
1522 video_stream_encoder_->Stop();
1523}
1524
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001525TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1526 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001527 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1528 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001529
1530 VideoEncoderConfig video_encoder_config;
1531 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1532 video_encoder_config.max_bitrate_bps = 0;
1533 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1534 kMaxPayloadLength);
1535
1536 // Encode 720p frame to get the default encoder target bitrate.
1537 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1538 WaitForEncodedFrame(1);
1539 const uint32_t kDefaultTargetBitrateFor720pKbps =
1540 bitrate_allocator_factory_.codec_config()
1541 .simulcastStream[0]
1542 .targetBitrate;
1543
1544 // Set the max recommended encoder bitrate to something lower than the default
1545 // target bitrate.
1546 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1547 1280 * 720, 10 * 1000, 10 * 1000,
1548 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1549 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1550
1551 // Change resolution to trigger encoder reinitialization.
1552 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1553 WaitForEncodedFrame(2);
1554 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1555 WaitForEncodedFrame(3);
1556
1557 // Ensure the target bitrate is capped by the max bitrate.
1558 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1559 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1560 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1561 .simulcastStream[0]
1562 .targetBitrate *
1563 1000,
1564 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1565
1566 video_stream_encoder_->Stop();
1567}
1568
mflodmancc3d4422017-08-03 08:27:51 -07001569TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001570 EXPECT_TRUE(video_source_.has_sinks());
1571 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001572 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001573 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001574 EXPECT_FALSE(video_source_.has_sinks());
1575 EXPECT_TRUE(new_video_source.has_sinks());
1576
mflodmancc3d4422017-08-03 08:27:51 -07001577 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001578}
1579
mflodmancc3d4422017-08-03 08:27:51 -07001580TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001581 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001582 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001583 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001584 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001585}
1586
Jonathan Yubc771b72017-12-08 17:04:29 -08001587TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1588 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001589 const int kWidth = 1280;
1590 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001591
1592 // We rely on the automatic resolution adaptation, but we handle framerate
1593 // adaptation manually by mocking the stats proxy.
1594 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001595
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001596 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001597 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001598 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1599 DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001600 video_stream_encoder_->SetSource(&video_source_,
1601 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001602 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001603 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001604 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001605 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1606
Jonathan Yubc771b72017-12-08 17:04:29 -08001607 // Adapt down as far as possible.
1608 rtc::VideoSinkWants last_wants;
1609 int64_t t = 1;
1610 int loop_count = 0;
1611 do {
1612 ++loop_count;
1613 last_wants = video_source_.sink_wants();
1614
1615 // Simulate the framerate we've been asked to adapt to.
1616 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1617 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1618 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1619 mock_stats.input_frame_rate = fps;
1620 stats_proxy_->SetMockStats(mock_stats);
1621
1622 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1623 sink_.WaitForEncodedFrame(t);
1624 t += frame_interval_ms;
1625
mflodmancc3d4422017-08-03 08:27:51 -07001626 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001627 VerifyBalancedModeFpsRange(
1628 video_source_.sink_wants(),
1629 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1630 } while (video_source_.sink_wants().max_pixel_count <
1631 last_wants.max_pixel_count ||
1632 video_source_.sink_wants().max_framerate_fps <
1633 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001634
Jonathan Yubc771b72017-12-08 17:04:29 -08001635 // Verify that we've adapted all the way down.
1636 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001637 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001638 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1639 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001640 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001641 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1642 *video_source_.last_sent_height());
1643 EXPECT_EQ(kMinBalancedFramerateFps,
1644 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001645
Jonathan Yubc771b72017-12-08 17:04:29 -08001646 // Adapt back up the same number of times we adapted down.
1647 for (int i = 0; i < loop_count - 1; ++i) {
1648 last_wants = video_source_.sink_wants();
1649
1650 // Simulate the framerate we've been asked to adapt to.
1651 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1652 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1653 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1654 mock_stats.input_frame_rate = fps;
1655 stats_proxy_->SetMockStats(mock_stats);
1656
1657 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1658 sink_.WaitForEncodedFrame(t);
1659 t += frame_interval_ms;
1660
mflodmancc3d4422017-08-03 08:27:51 -07001661 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001662 VerifyBalancedModeFpsRange(
1663 video_source_.sink_wants(),
1664 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1665 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1666 last_wants.max_pixel_count ||
1667 video_source_.sink_wants().max_framerate_fps >
1668 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001669 }
1670
Åsa Persson8c1bf952018-09-13 10:42:19 +02001671 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001672 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001673 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1675 EXPECT_EQ((loop_count - 1) * 2,
1676 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001677
mflodmancc3d4422017-08-03 08:27:51 -07001678 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001679}
mflodmancc3d4422017-08-03 08:27:51 -07001680TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001681 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001682 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1683 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001684 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001685
sprangc5d62e22017-04-02 23:53:04 -07001686 const int kFrameWidth = 1280;
1687 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001688
Åsa Persson8c1bf952018-09-13 10:42:19 +02001689 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001690
kthelgason5e13d412016-12-01 03:59:51 -08001691 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001692 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001693 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001694 frame_timestamp += kFrameIntervalMs;
1695
perkj803d97f2016-11-01 11:45:46 -07001696 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001697 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001698 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001699 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001700 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001701 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001702
asapersson0944a802017-04-07 00:57:58 -07001703 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001704 // wanted resolution.
1705 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1706 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1707 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001708 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001709
1710 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001711 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001712 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001713 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001714
sprangc5d62e22017-04-02 23:53:04 -07001715 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001716 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001717
sprangc5d62e22017-04-02 23:53:04 -07001718 // Force an input frame rate to be available, or the adaptation call won't
1719 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001720 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001721 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001722 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001723 stats_proxy_->SetMockStats(stats);
1724
mflodmancc3d4422017-08-03 08:27:51 -07001725 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001726 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001727 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001728 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001729 frame_timestamp += kFrameIntervalMs;
1730
1731 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001732 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001733 EXPECT_EQ(std::numeric_limits<int>::max(),
1734 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001735 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001736
asapersson02465b82017-04-10 01:12:52 -07001737 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001738 video_stream_encoder_->SetSource(&new_video_source,
1739 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001740 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001741
mflodmancc3d4422017-08-03 08:27:51 -07001742 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001743 new_video_source.IncomingCapturedFrame(
1744 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001745 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001746 frame_timestamp += kFrameIntervalMs;
1747
1748 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001749 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001750
1751 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001752 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001753 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001754 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1755 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001756 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001757 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001758
1759 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001760 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001761 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001762 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1763 EXPECT_EQ(std::numeric_limits<int>::max(),
1764 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001765 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001766
mflodmancc3d4422017-08-03 08:27:51 -07001767 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001768}
1769
mflodmancc3d4422017-08-03 08:27:51 -07001770TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001771 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001772 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1773 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001774
asaperssonfab67072017-04-04 05:51:49 -07001775 const int kWidth = 1280;
1776 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001777 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001778 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001779 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1780 EXPECT_FALSE(stats.bw_limited_resolution);
1781 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1782
1783 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001784 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001785 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001786 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001787
1788 stats = stats_proxy_->GetStats();
1789 EXPECT_TRUE(stats.bw_limited_resolution);
1790 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1791
1792 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001793 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001794 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001795 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001796
1797 stats = stats_proxy_->GetStats();
1798 EXPECT_FALSE(stats.bw_limited_resolution);
1799 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1800 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1801
mflodmancc3d4422017-08-03 08:27:51 -07001802 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001803}
1804
mflodmancc3d4422017-08-03 08:27:51 -07001805TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001806 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001807 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1808 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001809
1810 const int kWidth = 1280;
1811 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001812 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001813 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001814 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1815 EXPECT_FALSE(stats.cpu_limited_resolution);
1816 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1817
1818 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001819 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001820 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001821 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001822
1823 stats = stats_proxy_->GetStats();
1824 EXPECT_TRUE(stats.cpu_limited_resolution);
1825 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1826
1827 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001828 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001829 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001830 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001831
1832 stats = stats_proxy_->GetStats();
1833 EXPECT_FALSE(stats.cpu_limited_resolution);
1834 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001835 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001836
mflodmancc3d4422017-08-03 08:27:51 -07001837 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001838}
1839
mflodmancc3d4422017-08-03 08:27:51 -07001840TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001841 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001842 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1843 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001844
asaperssonfab67072017-04-04 05:51:49 -07001845 const int kWidth = 1280;
1846 const int kHeight = 720;
1847 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001848 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001849 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001850 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001851 EXPECT_FALSE(stats.cpu_limited_resolution);
1852 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1853
asaperssonfab67072017-04-04 05:51:49 -07001854 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001855 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001856 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001857 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001858 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001859 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001860 EXPECT_TRUE(stats.cpu_limited_resolution);
1861 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1862
1863 // Set new source with adaptation still enabled.
1864 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001865 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001866 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001867
asaperssonfab67072017-04-04 05:51:49 -07001868 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001869 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001870 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001871 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001872 EXPECT_TRUE(stats.cpu_limited_resolution);
1873 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1874
1875 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001876 video_stream_encoder_->SetSource(&new_video_source,
1877 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001878
asaperssonfab67072017-04-04 05:51:49 -07001879 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001880 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001881 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001882 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001883 EXPECT_FALSE(stats.cpu_limited_resolution);
1884 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1885
1886 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001887 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001888 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001889
asaperssonfab67072017-04-04 05:51:49 -07001890 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001891 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001892 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001893 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001894 EXPECT_TRUE(stats.cpu_limited_resolution);
1895 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1896
asaperssonfab67072017-04-04 05:51:49 -07001897 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001898 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001899 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001900 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001901 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001902 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001903 EXPECT_FALSE(stats.cpu_limited_resolution);
1904 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001905 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001906
mflodmancc3d4422017-08-03 08:27:51 -07001907 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001908}
1909
mflodmancc3d4422017-08-03 08:27:51 -07001910TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001911 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001912 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1913 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001914
asaperssonfab67072017-04-04 05:51:49 -07001915 const int kWidth = 1280;
1916 const int kHeight = 720;
1917 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001918 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001919 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001920 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001921 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001922 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001923
1924 // Set new source with adaptation still enabled.
1925 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001926 video_stream_encoder_->SetSource(&new_video_source,
1927 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001928
asaperssonfab67072017-04-04 05:51:49 -07001929 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001930 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001931 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001932 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001933 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001934 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001935
asaperssonfab67072017-04-04 05:51:49 -07001936 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001937 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001938 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001939 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001940 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001941 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001942 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001943 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001944
asaperssonfab67072017-04-04 05:51:49 -07001945 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001946 video_stream_encoder_->SetSource(&new_video_source,
1947 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001948
asaperssonfab67072017-04-04 05:51:49 -07001949 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001950 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001951 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001952 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001953 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001954 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001955
asapersson02465b82017-04-10 01:12:52 -07001956 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001957 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001958 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001959
asaperssonfab67072017-04-04 05:51:49 -07001960 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001961 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001962 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001963 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001964 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001965 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1966 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001967
mflodmancc3d4422017-08-03 08:27:51 -07001968 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001969}
1970
mflodmancc3d4422017-08-03 08:27:51 -07001971TEST_F(VideoStreamEncoderTest,
1972 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001973 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001974 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1975 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001976
1977 const int kWidth = 1280;
1978 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001979 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001980 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001981 video_source_.IncomingCapturedFrame(
1982 CreateFrame(timestamp_ms, kWidth, kHeight));
1983 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001984 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1985 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1986 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1987
1988 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001989 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001990 timestamp_ms += kFrameIntervalMs;
1991 video_source_.IncomingCapturedFrame(
1992 CreateFrame(timestamp_ms, kWidth, kHeight));
1993 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001994 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1995 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1996 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1997
1998 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001999 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002000 timestamp_ms += kFrameIntervalMs;
2001 video_source_.IncomingCapturedFrame(
2002 CreateFrame(timestamp_ms, kWidth, kHeight));
2003 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002004 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2005 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2006 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2007
Niels Möller4db138e2018-04-19 09:04:13 +02002008 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002009 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002010
2011 VideoEncoderConfig video_encoder_config;
2012 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2013 // Make format different, to force recreation of encoder.
2014 video_encoder_config.video_format.parameters["foo"] = "foo";
2015 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002016 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002017 timestamp_ms += kFrameIntervalMs;
2018 video_source_.IncomingCapturedFrame(
2019 CreateFrame(timestamp_ms, kWidth, kHeight));
2020 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002021 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2023 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2024
mflodmancc3d4422017-08-03 08:27:51 -07002025 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002026}
2027
mflodmancc3d4422017-08-03 08:27:51 -07002028TEST_F(VideoStreamEncoderTest,
2029 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002030 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002031 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2032 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002033
asapersson0944a802017-04-07 00:57:58 -07002034 const int kWidth = 1280;
2035 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002036 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002037
asaperssonfab67072017-04-04 05:51:49 -07002038 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002039 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002040 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002041 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002042 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002043 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2044
asapersson02465b82017-04-10 01:12:52 -07002045 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002046 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002047 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002048 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002049 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002050 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002051 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002052 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2053
2054 // Set new source with adaptation still enabled.
2055 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002056 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002057 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002058
2059 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002060 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002061 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002062 stats = stats_proxy_->GetStats();
2063 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002064 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002065 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2066
sprangc5d62e22017-04-02 23:53:04 -07002067 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002068 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002069 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002070 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002071 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002072 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002073 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002074 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002075 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002076 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002077 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2078
sprangc5d62e22017-04-02 23:53:04 -07002079 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002080 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002081 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2082 mock_stats.input_frame_rate = 30;
2083 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002084 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002085 stats_proxy_->ResetMockStats();
2086
2087 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002088 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002089 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002090
2091 // Framerate now adapted.
2092 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002093 EXPECT_FALSE(stats.cpu_limited_resolution);
2094 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002095 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2096
2097 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002098 video_stream_encoder_->SetSource(&new_video_source,
2099 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002100 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002101 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002102 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002103
2104 stats = stats_proxy_->GetStats();
2105 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002106 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002107 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2108
2109 // Try to trigger overuse. Should not succeed.
2110 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002111 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002112 stats_proxy_->ResetMockStats();
2113
2114 stats = stats_proxy_->GetStats();
2115 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002116 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002117 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2118
2119 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002120 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002121 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002122 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002123 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002124 stats = stats_proxy_->GetStats();
2125 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002126 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002127 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002128
2129 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002130 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002131 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002132 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002133 stats = stats_proxy_->GetStats();
2134 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002135 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002136 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2137
2138 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002139 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002140 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002141 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002142 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002143 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002144 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002145 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002146 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002147 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002148 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2149
2150 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002151 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002152 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002153 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002154 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002155 stats = stats_proxy_->GetStats();
2156 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002157 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002158 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002159 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002160
mflodmancc3d4422017-08-03 08:27:51 -07002161 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002162}
2163
mflodmancc3d4422017-08-03 08:27:51 -07002164TEST_F(VideoStreamEncoderTest,
2165 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002166 const int kWidth = 1280;
2167 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002168 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002169 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2170 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002171
asaperssonfab67072017-04-04 05:51:49 -07002172 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002173 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002174
asaperssonfab67072017-04-04 05:51:49 -07002175 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002176 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002177
asaperssonfab67072017-04-04 05:51:49 -07002178 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002179 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002180
asaperssonfab67072017-04-04 05:51:49 -07002181 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002182 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002183
kthelgason876222f2016-11-29 01:44:11 -08002184 // Expect a scale down.
2185 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002186 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002187
asapersson02465b82017-04-10 01:12:52 -07002188 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002189 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002190 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002191 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002192
asaperssonfab67072017-04-04 05:51:49 -07002193 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002194 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002195 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002196 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002197
asaperssonfab67072017-04-04 05:51:49 -07002198 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002199 EXPECT_EQ(std::numeric_limits<int>::max(),
2200 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002201
asaperssonfab67072017-04-04 05:51:49 -07002202 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002203 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002204 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002205 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002206
asapersson02465b82017-04-10 01:12:52 -07002207 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002208 EXPECT_EQ(std::numeric_limits<int>::max(),
2209 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002210
mflodmancc3d4422017-08-03 08:27:51 -07002211 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002212}
2213
mflodmancc3d4422017-08-03 08:27:51 -07002214TEST_F(VideoStreamEncoderTest,
2215 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002216 const int kWidth = 1280;
2217 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002218 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002219 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2220 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002221
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002222 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002223 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002224 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002225 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002226
2227 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002228 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002229 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002230 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2231 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2232
2233 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002234 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002235 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002236 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2237 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2238 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2239
2240 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002241 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002242 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2243 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2244 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2245
mflodmancc3d4422017-08-03 08:27:51 -07002246 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002247}
2248
mflodmancc3d4422017-08-03 08:27:51 -07002249TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002250 const int kWidth = 1280;
2251 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002252 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002253 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2254 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002255
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002256 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002257 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002258 video_stream_encoder_->SetSource(&source,
2259 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002260 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2261 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002262 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002263
2264 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002265 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002266 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2267 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2268 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2269 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2270
2271 // Trigger adapt down for same input resolution, expect no change.
2272 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2273 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002274 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002275 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2277 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2278
2279 // Trigger adapt down for larger input resolution, expect no change.
2280 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2281 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002282 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002283 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2284 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2285 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2286
mflodmancc3d4422017-08-03 08:27:51 -07002287 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002288}
2289
mflodmancc3d4422017-08-03 08:27:51 -07002290TEST_F(VideoStreamEncoderTest,
2291 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002292 const int kWidth = 1280;
2293 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002294 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002295 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2296 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002297
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002298 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002299 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002300 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002301 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002302
2303 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002304 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002305 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002306 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2307 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2308
2309 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002310 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002311 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002312 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2313 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2314
mflodmancc3d4422017-08-03 08:27:51 -07002315 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002316}
2317
mflodmancc3d4422017-08-03 08:27:51 -07002318TEST_F(VideoStreamEncoderTest,
2319 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002320 const int kWidth = 1280;
2321 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002322 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002323 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2324 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002325
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002326 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002327 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002328 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002329 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002330
2331 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002332 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002333 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002334 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002335 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2336
2337 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002338 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002339 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002340 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002341 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2342
mflodmancc3d4422017-08-03 08:27:51 -07002343 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002344}
2345
mflodmancc3d4422017-08-03 08:27:51 -07002346TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002347 const int kWidth = 1280;
2348 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002349 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002350 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2351 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002352
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002353 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002354 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002355 video_stream_encoder_->SetSource(&source,
2356 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002357
2358 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2359 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002360 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002361 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2362 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2363 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2364
2365 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002366 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002367 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002368 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2369 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2370 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2371
mflodmancc3d4422017-08-03 08:27:51 -07002372 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002373}
2374
mflodmancc3d4422017-08-03 08:27:51 -07002375TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002376 const int kWidth = 1280;
2377 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002378 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002379 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2380 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002381
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002382 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002383 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002384 video_stream_encoder_->SetSource(&source,
2385 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002386
2387 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2388 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002389 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2391 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2392 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2393
2394 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002395 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002396 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002397 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2398 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2399 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2400
mflodmancc3d4422017-08-03 08:27:51 -07002401 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002402}
2403
mflodmancc3d4422017-08-03 08:27:51 -07002404TEST_F(VideoStreamEncoderTest,
2405 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002406 const int kWidth = 1280;
2407 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002408 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002409 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2410 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002411
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002412 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002413 AdaptingFrameForwarder source;
2414 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002415 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002416 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002417
2418 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002419 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002420 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2422 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2423
2424 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002425 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002426 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002427 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002428 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002429 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2430 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2431
2432 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002433 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002434 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002435 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2436 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2437 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2438
mflodmancc3d4422017-08-03 08:27:51 -07002439 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002440}
2441
mflodmancc3d4422017-08-03 08:27:51 -07002442TEST_F(VideoStreamEncoderTest,
2443 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002444 const int kWidth = 1280;
2445 const int kHeight = 720;
2446 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002447 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002448 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2449 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002450
2451 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2452 stats.input_frame_rate = kInputFps;
2453 stats_proxy_->SetMockStats(stats);
2454
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002455 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002456 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2457 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002458 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002459
2460 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002461 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002462 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2463 sink_.WaitForEncodedFrame(2);
2464 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2465
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002466 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002467 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002468 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002469 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002470 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002471
2472 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002473 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002474 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2475 sink_.WaitForEncodedFrame(3);
2476 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2477
2478 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002479 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002480 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002481
mflodmancc3d4422017-08-03 08:27:51 -07002482 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002483}
2484
mflodmancc3d4422017-08-03 08:27:51 -07002485TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002486 const int kWidth = 1280;
2487 const int kHeight = 720;
2488 const size_t kNumFrames = 10;
2489
Erik Språng4c6ca302019-04-08 15:14:01 +02002490 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002491 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2492 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002493
asaperssond0de2952017-04-21 01:47:31 -07002494 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002495 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002496 video_source_.set_adaptation_enabled(true);
2497
2498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2499 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2500
2501 int downscales = 0;
2502 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002503 video_source_.IncomingCapturedFrame(
2504 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2505 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002506
asaperssonfab67072017-04-04 05:51:49 -07002507 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002508 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002509 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002510 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002511
2512 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2513 ++downscales;
2514
2515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2516 EXPECT_EQ(downscales,
2517 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2518 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002519 }
mflodmancc3d4422017-08-03 08:27:51 -07002520 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002521}
2522
mflodmancc3d4422017-08-03 08:27:51 -07002523TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002524 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2525 const int kWidth = 1280;
2526 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002527 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002528 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2529 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002530
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002531 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002532 AdaptingFrameForwarder source;
2533 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002534 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002535 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002536
Åsa Persson8c1bf952018-09-13 10:42:19 +02002537 int64_t timestamp_ms = kFrameIntervalMs;
2538 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002539 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002540 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002541 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2542 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2543
2544 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002545 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002546 timestamp_ms += kFrameIntervalMs;
2547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2548 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002549 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002550 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2551 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2552
2553 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002554 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002555 timestamp_ms += kFrameIntervalMs;
2556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002557 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002558 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002559 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2560 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2561
2562 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002563 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002564 timestamp_ms += kFrameIntervalMs;
2565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2566 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002567 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002568 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2569 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2570
2571 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002572 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002573 timestamp_ms += kFrameIntervalMs;
2574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002575 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002576 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002577 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2578 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2579
mflodmancc3d4422017-08-03 08:27:51 -07002580 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002581}
2582
mflodmancc3d4422017-08-03 08:27:51 -07002583TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002584 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2585 const int kWidth = 1280;
2586 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002587 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002588 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2589 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002590
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002591 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002592 AdaptingFrameForwarder source;
2593 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002594 video_stream_encoder_->SetSource(&source,
2595 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002596
Åsa Persson8c1bf952018-09-13 10:42:19 +02002597 int64_t timestamp_ms = kFrameIntervalMs;
2598 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002599 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002600 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2602 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2603
2604 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002605 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002606 timestamp_ms += kFrameIntervalMs;
2607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2608 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002609 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2611 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2612
2613 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002614 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002615 timestamp_ms += kFrameIntervalMs;
2616 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002617 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002618 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2620 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2621
2622 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002623 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002624 timestamp_ms += kFrameIntervalMs;
2625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2626 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002627 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2629 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2630
2631 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002633 timestamp_ms += kFrameIntervalMs;
2634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002635 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002636 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002637 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2638 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2639
mflodmancc3d4422017-08-03 08:27:51 -07002640 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002641}
2642
Åsa Persson1b247f12019-08-14 17:26:39 +02002643TEST_F(VideoStreamEncoderTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
2644 webrtc::test::ScopedFieldTrials field_trials(
2645 "WebRTC-Video-BalancedDegradationSettings/"
2646 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
2647 // Reset encoder for field trials to take effect.
2648 ConfigureEncoder(video_encoder_config_.Copy());
2649
2650 const int kWidth = 640; // pixels:640x360=230400
2651 const int kHeight = 360;
2652 const int64_t kFrameIntervalMs = 150;
2653 const int kMinBitrateBps = 425000;
2654 const int kTooLowMinBitrateBps = 424000;
Florent Castellia8336d32019-09-09 13:36:55 +02002655 video_stream_encoder_->OnBitrateUpdated(
2656 DataRate::bps(kTooLowMinBitrateBps), DataRate::bps(kTooLowMinBitrateBps),
2657 DataRate::bps(kTooLowMinBitrateBps), 0, 0);
Åsa Persson1b247f12019-08-14 17:26:39 +02002658
2659 // Enable BALANCED preference, no initial limitation.
2660 AdaptingFrameForwarder source;
2661 source.set_adaptation_enabled(true);
2662 video_stream_encoder_->SetSource(&source,
2663 webrtc::DegradationPreference::BALANCED);
2664
2665 int64_t timestamp_ms = kFrameIntervalMs;
2666 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2667 sink_.WaitForEncodedFrame(kWidth, kHeight);
2668 VerifyFpsMaxResolutionMax(source.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002669 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2670
2671 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2672 video_stream_encoder_->TriggerQualityLow();
2673 timestamp_ms += kFrameIntervalMs;
2674 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2675 sink_.WaitForEncodedFrame(timestamp_ms);
2676 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02002677 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2678
2679 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2680 video_stream_encoder_->TriggerQualityLow();
2681 timestamp_ms += kFrameIntervalMs;
2682 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2683 sink_.WaitForEncodedFrame(timestamp_ms);
2684 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002685 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2686
Åsa Persson30ab0152019-08-27 12:22:33 +02002687 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2688 video_stream_encoder_->TriggerQualityLow();
2689 timestamp_ms += kFrameIntervalMs;
2690 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2691 sink_.WaitForEncodedFrame(timestamp_ms);
2692 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2693 EXPECT_EQ(source.sink_wants().max_framerate_fps, 10);
2694 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2695
2696 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02002697 video_stream_encoder_->TriggerQualityHigh();
2698 timestamp_ms += kFrameIntervalMs;
2699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2700 sink_.WaitForEncodedFrame(timestamp_ms);
Åsa Persson30ab0152019-08-27 12:22:33 +02002701 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02002702
Åsa Persson30ab0152019-08-27 12:22:33 +02002703 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02002704 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002705 DataRate::bps(kMinBitrateBps),
Åsa Persson1b247f12019-08-14 17:26:39 +02002706 DataRate::bps(kMinBitrateBps), 0, 0);
2707 video_stream_encoder_->TriggerQualityHigh();
2708 timestamp_ms += kFrameIntervalMs;
2709 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2710 sink_.WaitForEncodedFrame(timestamp_ms);
Åsa Persson30ab0152019-08-27 12:22:33 +02002711 EXPECT_EQ(source.sink_wants().max_framerate_fps, 14);
2712 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2713
2714 video_stream_encoder_->Stop();
2715}
2716
2717TEST_F(VideoStreamEncoderTest,
2718 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
2719 webrtc::test::ScopedFieldTrials field_trials(
2720 "WebRTC-Video-BalancedDegradationSettings/"
2721 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
2722 // Reset encoder for field trials to take effect.
2723 ConfigureEncoder(video_encoder_config_.Copy());
2724
2725 const int kWidth = 640; // pixels:640x360=230400
2726 const int kHeight = 360;
2727 const int64_t kFrameIntervalMs = 150;
2728 const int kResolutionMinBitrateBps = 435000;
2729 const int kTooLowMinResolutionBitrateBps = 434000;
2730 video_stream_encoder_->OnBitrateUpdated(
2731 DataRate::bps(kTooLowMinResolutionBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002732 DataRate::bps(kTooLowMinResolutionBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002733 DataRate::bps(kTooLowMinResolutionBitrateBps), 0, 0);
2734
2735 // Enable BALANCED preference, no initial limitation.
2736 AdaptingFrameForwarder source;
2737 source.set_adaptation_enabled(true);
2738 video_stream_encoder_->SetSource(&source,
2739 webrtc::DegradationPreference::BALANCED);
2740
2741 int64_t timestamp_ms = kFrameIntervalMs;
2742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2743 sink_.WaitForEncodedFrame(kWidth, kHeight);
2744 VerifyFpsMaxResolutionMax(source.sink_wants());
2745 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2746
2747 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2748 video_stream_encoder_->TriggerQualityLow();
2749 timestamp_ms += kFrameIntervalMs;
2750 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2751 sink_.WaitForEncodedFrame(timestamp_ms);
2752 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2753 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2754
2755 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2756 video_stream_encoder_->TriggerQualityLow();
2757 timestamp_ms += kFrameIntervalMs;
2758 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2759 sink_.WaitForEncodedFrame(timestamp_ms);
2760 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2761 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2762
2763 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2764 video_stream_encoder_->TriggerQualityLow();
2765 timestamp_ms += kFrameIntervalMs;
2766 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2767 sink_.WaitForEncodedFrame(timestamp_ms);
2768 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002769 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2770
Åsa Persson30ab0152019-08-27 12:22:33 +02002771 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
2772 video_stream_encoder_->TriggerQualityHigh();
2773 timestamp_ms += kFrameIntervalMs;
2774 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2775 sink_.WaitForEncodedFrame(timestamp_ms);
2776 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2777 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2778
2779 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2780 video_stream_encoder_->TriggerQualityHigh();
2781 timestamp_ms += kFrameIntervalMs;
2782 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2783 sink_.WaitForEncodedFrame(timestamp_ms);
2784 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2785
2786 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
2787 video_stream_encoder_->OnBitrateUpdated(
2788 DataRate::bps(kResolutionMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002789 DataRate::bps(kResolutionMinBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002790 DataRate::bps(kResolutionMinBitrateBps), 0, 0);
2791 video_stream_encoder_->TriggerQualityHigh();
2792 timestamp_ms += kFrameIntervalMs;
2793 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2794 sink_.WaitForEncodedFrame(timestamp_ms);
2795 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2796 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2797
2798 video_stream_encoder_->Stop();
2799}
2800
2801TEST_F(VideoStreamEncoderTest,
2802 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
2803 webrtc::test::ScopedFieldTrials field_trials(
2804 "WebRTC-Video-BalancedDegradationSettings/"
2805 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
2806 // Reset encoder for field trials to take effect.
2807 ConfigureEncoder(video_encoder_config_.Copy());
2808
2809 const int kWidth = 640; // pixels:640x360=230400
2810 const int kHeight = 360;
2811 const int64_t kFrameIntervalMs = 150;
2812 const int kMinBitrateBps = 425000;
2813 const int kTooLowMinBitrateBps = 424000;
2814 const int kResolutionMinBitrateBps = 435000;
2815 const int kTooLowMinResolutionBitrateBps = 434000;
Florent Castellia8336d32019-09-09 13:36:55 +02002816 video_stream_encoder_->OnBitrateUpdated(
2817 DataRate::bps(kTooLowMinBitrateBps), DataRate::bps(kTooLowMinBitrateBps),
2818 DataRate::bps(kTooLowMinBitrateBps), 0, 0);
Åsa Persson30ab0152019-08-27 12:22:33 +02002819
2820 // Enable BALANCED preference, no initial limitation.
2821 AdaptingFrameForwarder source;
2822 source.set_adaptation_enabled(true);
2823 video_stream_encoder_->SetSource(&source,
2824 webrtc::DegradationPreference::BALANCED);
2825
2826 int64_t timestamp_ms = kFrameIntervalMs;
2827 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2828 sink_.WaitForEncodedFrame(kWidth, kHeight);
2829 VerifyFpsMaxResolutionMax(source.sink_wants());
2830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2831
2832 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2833 video_stream_encoder_->TriggerQualityLow();
2834 timestamp_ms += kFrameIntervalMs;
2835 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2836 sink_.WaitForEncodedFrame(timestamp_ms);
2837 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2838 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2839
2840 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2841 video_stream_encoder_->TriggerQualityLow();
2842 timestamp_ms += kFrameIntervalMs;
2843 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2844 sink_.WaitForEncodedFrame(timestamp_ms);
2845 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2846 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2847
2848 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2849 video_stream_encoder_->TriggerQualityLow();
2850 timestamp_ms += kFrameIntervalMs;
2851 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2852 sink_.WaitForEncodedFrame(timestamp_ms);
2853 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2854 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2855
2856 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
2857 video_stream_encoder_->TriggerQualityHigh();
2858 timestamp_ms += kFrameIntervalMs;
2859 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2860 sink_.WaitForEncodedFrame(timestamp_ms);
2861 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2862
2863 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
2864 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002865 DataRate::bps(kMinBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002866 DataRate::bps(kMinBitrateBps), 0, 0);
2867 video_stream_encoder_->TriggerQualityHigh();
2868 timestamp_ms += kFrameIntervalMs;
2869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2870 sink_.WaitForEncodedFrame(timestamp_ms);
2871 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2872 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2873
2874 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2875 video_stream_encoder_->OnBitrateUpdated(
2876 DataRate::bps(kTooLowMinResolutionBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002877 DataRate::bps(kTooLowMinResolutionBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002878 DataRate::bps(kTooLowMinResolutionBitrateBps), 0, 0);
2879 video_stream_encoder_->TriggerQualityHigh();
2880 timestamp_ms += kFrameIntervalMs;
2881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2882 sink_.WaitForEncodedFrame(timestamp_ms);
2883 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2884
2885 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
2886 video_stream_encoder_->OnBitrateUpdated(
2887 DataRate::bps(kResolutionMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002888 DataRate::bps(kResolutionMinBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002889 DataRate::bps(kResolutionMinBitrateBps), 0, 0);
2890 video_stream_encoder_->TriggerQualityHigh();
2891 timestamp_ms += kFrameIntervalMs;
2892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2893 sink_.WaitForEncodedFrame(timestamp_ms);
2894 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2895 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2896
Åsa Persson1b247f12019-08-14 17:26:39 +02002897 video_stream_encoder_->Stop();
2898}
2899
mflodmancc3d4422017-08-03 08:27:51 -07002900TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002901 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2902 const int kWidth = 1280;
2903 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002904 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002905 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2906 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002907
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002908 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002909 AdaptingFrameForwarder source;
2910 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002911 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002912 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002913
Åsa Persson8c1bf952018-09-13 10:42:19 +02002914 int64_t timestamp_ms = kFrameIntervalMs;
2915 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002916 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002917 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002918 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2920 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2921 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2922
2923 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002924 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002925 timestamp_ms += kFrameIntervalMs;
2926 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2927 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002928 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002929 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2930 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2931 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2932 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2933
2934 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002935 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002936 timestamp_ms += kFrameIntervalMs;
2937 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2938 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002939 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002940 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2941 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2942 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2943 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2944
Jonathan Yubc771b72017-12-08 17:04:29 -08002945 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002946 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002947 timestamp_ms += kFrameIntervalMs;
2948 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2949 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002950 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002951 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2952 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002953 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002954 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2955
Jonathan Yubc771b72017-12-08 17:04:29 -08002956 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002957 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002958 timestamp_ms += kFrameIntervalMs;
2959 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2960 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002961 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002962 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002963 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2964 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2965 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2966 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2967
Jonathan Yubc771b72017-12-08 17:04:29 -08002968 // Trigger quality adapt down, expect no change (min resolution reached).
2969 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002970 timestamp_ms += kFrameIntervalMs;
2971 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2972 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002973 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2974 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2975 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2976 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2977 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2978
2979 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002980 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002981 timestamp_ms += kFrameIntervalMs;
2982 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2983 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002984 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002985 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2986 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2987 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2988 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2989
2990 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2991 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002992 timestamp_ms += kFrameIntervalMs;
2993 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2994 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002995 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2996 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2997 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2998 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2999 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3000
3001 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3002 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003003 timestamp_ms += kFrameIntervalMs;
3004 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3005 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003006 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003007 last_wants = source.sink_wants();
3008 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3009 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003010 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003011 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3012
3013 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003014 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003015 timestamp_ms += kFrameIntervalMs;
3016 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3017 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003018 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003019 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3020 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003021 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003022 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3023
3024 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003025 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003026 timestamp_ms += kFrameIntervalMs;
3027 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003028 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003029 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003030 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003031 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003033 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003034 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003035
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003037}
3038
mflodmancc3d4422017-08-03 08:27:51 -07003039TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003040 const int kWidth = 640;
3041 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003042
Erik Språng4c6ca302019-04-08 15:14:01 +02003043 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003044 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3045 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003046
perkj803d97f2016-11-01 11:45:46 -07003047 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003048 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003049 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003050 }
3051
mflodmancc3d4422017-08-03 08:27:51 -07003052 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003053 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003054 video_source_.IncomingCapturedFrame(CreateFrame(
3055 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003056 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003057 }
3058
mflodmancc3d4422017-08-03 08:27:51 -07003059 video_stream_encoder_->Stop();
3060 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003061 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003062
perkj803d97f2016-11-01 11:45:46 -07003063 EXPECT_EQ(1,
3064 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3065 EXPECT_EQ(
3066 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3067}
3068
mflodmancc3d4422017-08-03 08:27:51 -07003069TEST_F(VideoStreamEncoderTest,
3070 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003071 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003072 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3073 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003074 const int kWidth = 640;
3075 const int kHeight = 360;
3076
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003077 video_stream_encoder_->SetSource(&video_source_,
3078 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003079
3080 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3081 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003082 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003083 }
3084
mflodmancc3d4422017-08-03 08:27:51 -07003085 video_stream_encoder_->Stop();
3086 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003087 stats_proxy_.reset();
3088
3089 EXPECT_EQ(0,
3090 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3091}
3092
mflodmancc3d4422017-08-03 08:27:51 -07003093TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003094 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003095 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003096
3097 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003098 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08003099 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003100 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3101 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003102
sprang57c2fff2017-01-16 06:24:02 -08003103 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003104 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003105 video_stream_encoder_->OnBitrateUpdated(
3106 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3107 DataRate::bps(kLowTargetBitrateBps), 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003108
sprang57c2fff2017-01-16 06:24:02 -08003109 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003110 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3111 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003112 VideoBitrateAllocation bitrate_allocation =
3113 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003114 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003115 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003116 // TODO(srte): The use of millisecs here looks like an error, but the tests
3117 // fails using seconds, this should be investigated.
3118 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003119
3120 // Not called on second frame.
3121 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3122 .Times(0);
3123 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003124 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3125 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02003126 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003127
3128 // Called after a process interval.
3129 const int64_t kProcessIntervalMs =
3130 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08003131 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3132 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003133 const int64_t start_time_ms = rtc::TimeMillis();
3134 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3135 video_source_.IncomingCapturedFrame(
3136 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3137 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02003138 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003139 }
3140
3141 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003142 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003143
mflodmancc3d4422017-08-03 08:27:51 -07003144 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003145}
3146
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003147TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3148 // 2 TLs configured, temporal layers supported by encoder.
3149 const int kNumTemporalLayers = 2;
3150 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3151 fake_encoder_.SetTemporalLayersSupported(0, true);
3152
3153 // Bitrate allocated across temporal layers.
3154 const int kTl0Bps = kTargetBitrateBps *
3155 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3156 kNumTemporalLayers, /*temporal_id*/ 0);
3157 const int kTl1Bps = kTargetBitrateBps *
3158 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3159 kNumTemporalLayers, /*temporal_id*/ 1);
3160 VideoBitrateAllocation expected_bitrate;
3161 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3162 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3163
3164 VerifyAllocatedBitrate(expected_bitrate);
3165 video_stream_encoder_->Stop();
3166}
3167
3168TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3169 // 2 TLs configured, temporal layers not supported by encoder.
3170 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3171 fake_encoder_.SetTemporalLayersSupported(0, false);
3172
3173 // Temporal layers not supported by the encoder.
3174 // Total bitrate should be at ti:0.
3175 VideoBitrateAllocation expected_bitrate;
3176 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3177
3178 VerifyAllocatedBitrate(expected_bitrate);
3179 video_stream_encoder_->Stop();
3180}
3181
3182TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3183 // 2 TLs configured, temporal layers only supported for first stream.
3184 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3185 fake_encoder_.SetTemporalLayersSupported(0, true);
3186 fake_encoder_.SetTemporalLayersSupported(1, false);
3187
3188 const int kS0Bps = 150000;
3189 const int kS0Tl0Bps =
3190 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3191 /*num_layers*/ 2, /*temporal_id*/ 0);
3192 const int kS0Tl1Bps =
3193 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3194 /*num_layers*/ 2, /*temporal_id*/ 1);
3195 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3196 // Temporal layers not supported by si:1.
3197 VideoBitrateAllocation expected_bitrate;
3198 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3199 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3200 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3201
3202 VerifyAllocatedBitrate(expected_bitrate);
3203 video_stream_encoder_->Stop();
3204}
3205
Niels Möller7dc26b72017-12-06 10:27:48 +01003206TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3207 const int kFrameWidth = 1280;
3208 const int kFrameHeight = 720;
3209 const int kFramerate = 24;
3210
Erik Språng4c6ca302019-04-08 15:14:01 +02003211 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003212 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3213 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003214 test::FrameForwarder source;
3215 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003216 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003217
3218 // Insert a single frame, triggering initial configuration.
3219 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3220 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3221
3222 EXPECT_EQ(
3223 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3224 kDefaultFramerate);
3225
3226 // Trigger reconfigure encoder (without resetting the entire instance).
3227 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003228 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003229 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3230 video_encoder_config.number_of_streams = 1;
3231 video_encoder_config.video_stream_factory =
3232 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3233 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003234 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003235 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3236
3237 // Detector should be updated with fps limit from codec config.
3238 EXPECT_EQ(
3239 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3240 kFramerate);
3241
3242 // Trigger overuse, max framerate should be reduced.
3243 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3244 stats.input_frame_rate = kFramerate;
3245 stats_proxy_->SetMockStats(stats);
3246 video_stream_encoder_->TriggerCpuOveruse();
3247 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3248 int adapted_framerate =
3249 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3250 EXPECT_LT(adapted_framerate, kFramerate);
3251
3252 // Trigger underuse, max framerate should go back to codec configured fps.
3253 // Set extra low fps, to make sure it's actually reset, not just incremented.
3254 stats = stats_proxy_->GetStats();
3255 stats.input_frame_rate = adapted_framerate / 2;
3256 stats_proxy_->SetMockStats(stats);
3257 video_stream_encoder_->TriggerCpuNormalUsage();
3258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3259 EXPECT_EQ(
3260 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3261 kFramerate);
3262
3263 video_stream_encoder_->Stop();
3264}
3265
3266TEST_F(VideoStreamEncoderTest,
3267 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3268 const int kFrameWidth = 1280;
3269 const int kFrameHeight = 720;
3270 const int kLowFramerate = 15;
3271 const int kHighFramerate = 25;
3272
Erik Språng4c6ca302019-04-08 15:14:01 +02003273 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003274 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3275 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003276 test::FrameForwarder source;
3277 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003278 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003279
3280 // Trigger initial configuration.
3281 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003282 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003283 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3284 video_encoder_config.number_of_streams = 1;
3285 video_encoder_config.video_stream_factory =
3286 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3287 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3288 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003289 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3291
3292 EXPECT_EQ(
3293 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3294 kLowFramerate);
3295
3296 // Trigger overuse, max framerate should be reduced.
3297 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3298 stats.input_frame_rate = kLowFramerate;
3299 stats_proxy_->SetMockStats(stats);
3300 video_stream_encoder_->TriggerCpuOveruse();
3301 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3302 int adapted_framerate =
3303 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3304 EXPECT_LT(adapted_framerate, kLowFramerate);
3305
3306 // Reconfigure the encoder with a new (higher max framerate), max fps should
3307 // still respect the adaptation.
3308 video_encoder_config.video_stream_factory =
3309 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3310 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3311 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003312 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3314
3315 EXPECT_EQ(
3316 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3317 adapted_framerate);
3318
3319 // Trigger underuse, max framerate should go back to codec configured fps.
3320 stats = stats_proxy_->GetStats();
3321 stats.input_frame_rate = adapted_framerate;
3322 stats_proxy_->SetMockStats(stats);
3323 video_stream_encoder_->TriggerCpuNormalUsage();
3324 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3325 EXPECT_EQ(
3326 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3327 kHighFramerate);
3328
3329 video_stream_encoder_->Stop();
3330}
3331
mflodmancc3d4422017-08-03 08:27:51 -07003332TEST_F(VideoStreamEncoderTest,
3333 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003334 const int kFrameWidth = 1280;
3335 const int kFrameHeight = 720;
3336 const int kFramerate = 24;
3337
Erik Språng4c6ca302019-04-08 15:14:01 +02003338 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003339 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3340 DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003341 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003342 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003343 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003344
3345 // Trigger initial configuration.
3346 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003347 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003348 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3349 video_encoder_config.number_of_streams = 1;
3350 video_encoder_config.video_stream_factory =
3351 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3352 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003353 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003354 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003355 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003356
Niels Möller7dc26b72017-12-06 10:27:48 +01003357 EXPECT_EQ(
3358 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3359 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003360
3361 // Trigger overuse, max framerate should be reduced.
3362 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3363 stats.input_frame_rate = kFramerate;
3364 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003365 video_stream_encoder_->TriggerCpuOveruse();
3366 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003367 int adapted_framerate =
3368 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003369 EXPECT_LT(adapted_framerate, kFramerate);
3370
3371 // Change degradation preference to not enable framerate scaling. Target
3372 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003373 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003374 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003375 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003376 EXPECT_EQ(
3377 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3378 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003379
mflodmancc3d4422017-08-03 08:27:51 -07003380 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003381}
3382
mflodmancc3d4422017-08-03 08:27:51 -07003383TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003384 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003385 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003386 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003387 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003388 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003389 const int kWidth = 640;
3390 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003391
asaperssonfab67072017-04-04 05:51:49 -07003392 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003393
3394 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003395 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003396
3397 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003398 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003399
sprangc5d62e22017-04-02 23:53:04 -07003400 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003401
asaperssonfab67072017-04-04 05:51:49 -07003402 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003403 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003404 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003405
3406 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003407 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003408
sprangc5d62e22017-04-02 23:53:04 -07003409 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003410
mflodmancc3d4422017-08-03 08:27:51 -07003411 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003412}
3413
mflodmancc3d4422017-08-03 08:27:51 -07003414TEST_F(VideoStreamEncoderTest,
3415 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003416 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003417 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003418 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003419 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003420 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003421 const int kWidth = 640;
3422 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003423
3424 // We expect the n initial frames to get dropped.
3425 int i;
3426 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003427 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003428 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003429 }
3430 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003431 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003432 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003433
3434 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003435 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003436
mflodmancc3d4422017-08-03 08:27:51 -07003437 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003438}
3439
mflodmancc3d4422017-08-03 08:27:51 -07003440TEST_F(VideoStreamEncoderTest,
3441 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003442 const int kWidth = 640;
3443 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003444 video_stream_encoder_->OnBitrateUpdated(
3445 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3446 DataRate::bps(kLowTargetBitrateBps), 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003447
3448 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003449 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003450 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003451
asaperssonfab67072017-04-04 05:51:49 -07003452 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003453 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003454 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003455
mflodmancc3d4422017-08-03 08:27:51 -07003456 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003457}
3458
mflodmancc3d4422017-08-03 08:27:51 -07003459TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003460 const int kWidth = 640;
3461 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003462 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003463
3464 VideoEncoderConfig video_encoder_config;
3465 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3466 // Make format different, to force recreation of encoder.
3467 video_encoder_config.video_format.parameters["foo"] = "foo";
3468 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003469 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003470 video_stream_encoder_->OnBitrateUpdated(
3471 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3472 DataRate::bps(kLowTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003473
kthelgasonb83797b2017-02-14 11:57:25 -08003474 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003475 video_stream_encoder_->SetSource(&video_source_,
3476 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003477
asaperssonfab67072017-04-04 05:51:49 -07003478 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003479 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003480 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003481
mflodmancc3d4422017-08-03 08:27:51 -07003482 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003483 fake_encoder_.SetQualityScaling(true);
3484}
3485
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003486TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
3487 webrtc::test::ScopedFieldTrials field_trials(
3488 "WebRTC-InitialFramedrop/Enabled/");
3489 // Reset encoder for field trials to take effect.
3490 ConfigureEncoder(video_encoder_config_.Copy());
3491 const int kTooLowBitrateForFrameSizeBps = 10000;
3492 const int kWidth = 640;
3493 const int kHeight = 360;
3494
Erik Språng4c6ca302019-04-08 15:14:01 +02003495 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003496 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3497 DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003498 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3499 // Frame should not be dropped.
3500 WaitForEncodedFrame(1);
3501
Erik Språng610c7632019-03-06 15:37:33 +01003502 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003503 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003504 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003505 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003506 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3507 // Expect to drop this frame, the wait should time out.
3508 ExpectDroppedFrame();
3509
3510 // Expect the sink_wants to specify a scaled frame.
3511 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3512 video_stream_encoder_->Stop();
3513}
3514
Åsa Persson139f4dc2019-08-02 09:29:58 +02003515TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3516 webrtc::test::ScopedFieldTrials field_trials(
3517 "WebRTC-Video-QualityScalerSettings/"
3518 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3519 // Reset encoder for field trials to take effect.
3520 ConfigureEncoder(video_encoder_config_.Copy());
3521 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3522 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3523 const int kWidth = 640;
3524 const int kHeight = 360;
3525
3526 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003527 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3528 DataRate::bps(kTargetBitrateBps), 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003529 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3530 // Frame should not be dropped.
3531 WaitForEncodedFrame(1);
3532
3533 video_stream_encoder_->OnBitrateUpdated(
3534 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003535 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Åsa Persson139f4dc2019-08-02 09:29:58 +02003536 DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0);
3537 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3538 // Frame should not be dropped.
3539 WaitForEncodedFrame(2);
3540
3541 video_stream_encoder_->OnBitrateUpdated(
3542 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003543 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Åsa Persson139f4dc2019-08-02 09:29:58 +02003544 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
3545 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3546 // Expect to drop this frame, the wait should time out.
3547 ExpectDroppedFrame();
3548
3549 // Expect the sink_wants to specify a scaled frame.
3550 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3551 video_stream_encoder_->Stop();
3552}
3553
mflodmancc3d4422017-08-03 08:27:51 -07003554TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003555 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3556 const int kTooSmallWidth = 10;
3557 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02003558 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003559 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3560 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003561
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003562 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003563 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003564 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003565 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003566 VerifyNoLimitation(source.sink_wants());
3567 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3568
3569 // Trigger adapt down, too small frame, expect no change.
3570 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003571 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003572 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003573 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003574 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3575 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3576
mflodmancc3d4422017-08-03 08:27:51 -07003577 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003578}
3579
mflodmancc3d4422017-08-03 08:27:51 -07003580TEST_F(VideoStreamEncoderTest,
3581 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003582 const int kTooSmallWidth = 10;
3583 const int kTooSmallHeight = 10;
3584 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02003585 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003586 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3587 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003588
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003589 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003590 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003591 video_stream_encoder_->SetSource(&source,
3592 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003593 VerifyNoLimitation(source.sink_wants());
3594 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3596
3597 // Trigger adapt down, expect limited framerate.
3598 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003599 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003600 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003601 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3602 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3604 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3605
3606 // Trigger adapt down, too small frame, expect no change.
3607 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003608 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003609 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003610 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3611 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3613 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3614
mflodmancc3d4422017-08-03 08:27:51 -07003615 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003616}
3617
mflodmancc3d4422017-08-03 08:27:51 -07003618TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07003619 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02003620 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003621 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3622 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02003623 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07003624 const int kFrameWidth = 1280;
3625 const int kFrameHeight = 720;
3626 video_source_.IncomingCapturedFrame(
3627 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003628 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07003629 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003630}
3631
sprangb1ca0732017-02-01 08:38:12 -08003632// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07003633TEST_F(VideoStreamEncoderTest,
3634 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003635 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003636 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3637 DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08003638
3639 const int kFrameWidth = 1280;
3640 const int kFrameHeight = 720;
3641 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07003642 // requested by
3643 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08003644 video_source_.set_adaptation_enabled(true);
3645
3646 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003647 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003648 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003649
3650 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003651 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08003652 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003653 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003654 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08003655
asaperssonfab67072017-04-04 05:51:49 -07003656 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003657 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08003658 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003659 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003660 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003661
mflodmancc3d4422017-08-03 08:27:51 -07003662 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08003663}
sprangfe627f32017-03-29 08:24:59 -07003664
mflodmancc3d4422017-08-03 08:27:51 -07003665TEST_F(VideoStreamEncoderTest,
3666 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07003667 const int kFrameWidth = 1280;
3668 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07003669
Erik Språng4c6ca302019-04-08 15:14:01 +02003670 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003671 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3672 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003673 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003674 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003675 video_source_.set_adaptation_enabled(true);
3676
sprang4847ae62017-06-27 07:06:52 -07003677 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003678
3679 video_source_.IncomingCapturedFrame(
3680 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003681 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003682
3683 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07003684 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003685
3686 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07003687 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003688 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003689 video_source_.IncomingCapturedFrame(
3690 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003691 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003692 }
3693
3694 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07003695 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003696 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003697 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003698 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003699 video_source_.IncomingCapturedFrame(
3700 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003701 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003702 ++num_frames_dropped;
3703 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003704 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003705 }
3706 }
3707
sprang4847ae62017-06-27 07:06:52 -07003708 // Add some slack to account for frames dropped by the frame dropper.
3709 const int kErrorMargin = 1;
3710 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003711 kErrorMargin);
3712
3713 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07003714 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003715 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003716 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003717 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003718 video_source_.IncomingCapturedFrame(
3719 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003720 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003721 ++num_frames_dropped;
3722 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003723 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003724 }
3725 }
sprang4847ae62017-06-27 07:06:52 -07003726 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07003727 kErrorMargin);
3728
3729 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07003730 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003731 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003732 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003733 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003734 video_source_.IncomingCapturedFrame(
3735 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003736 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003737 ++num_frames_dropped;
3738 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003739 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003740 }
3741 }
sprang4847ae62017-06-27 07:06:52 -07003742 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003743 kErrorMargin);
3744
3745 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07003746 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003747 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003748 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003749 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003750 video_source_.IncomingCapturedFrame(
3751 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003752 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003753 ++num_frames_dropped;
3754 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003755 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003756 }
3757 }
3758 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
3759
mflodmancc3d4422017-08-03 08:27:51 -07003760 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003761}
3762
mflodmancc3d4422017-08-03 08:27:51 -07003763TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07003764 const int kFramerateFps = 5;
3765 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07003766 const int kFrameWidth = 1280;
3767 const int kFrameHeight = 720;
3768
sprang4847ae62017-06-27 07:06:52 -07003769 // Reconfigure encoder with two temporal layers and screensharing, which will
3770 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02003771 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07003772
Erik Språng4c6ca302019-04-08 15:14:01 +02003773 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003774 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3775 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003776 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003777 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003778 video_source_.set_adaptation_enabled(true);
3779
sprang4847ae62017-06-27 07:06:52 -07003780 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003781
3782 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08003783 rtc::VideoSinkWants last_wants;
3784 do {
3785 last_wants = video_source_.sink_wants();
3786
sprangc5d62e22017-04-02 23:53:04 -07003787 // Insert frames to get a new fps estimate...
3788 for (int j = 0; j < kFramerateFps; ++j) {
3789 video_source_.IncomingCapturedFrame(
3790 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08003791 if (video_source_.last_sent_width()) {
3792 sink_.WaitForEncodedFrame(timestamp_ms);
3793 }
sprangc5d62e22017-04-02 23:53:04 -07003794 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003795 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07003796 }
3797 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07003798 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003799 } while (video_source_.sink_wants().max_framerate_fps <
3800 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003801
Jonathan Yubc771b72017-12-08 17:04:29 -08003802 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003803
mflodmancc3d4422017-08-03 08:27:51 -07003804 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003805}
asaperssonf7e294d2017-06-13 23:25:22 -07003806
mflodmancc3d4422017-08-03 08:27:51 -07003807TEST_F(VideoStreamEncoderTest,
3808 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003809 const int kWidth = 1280;
3810 const int kHeight = 720;
3811 const int64_t kFrameIntervalMs = 150;
3812 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003813 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003814 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3815 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003816
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003817 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003818 AdaptingFrameForwarder source;
3819 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003820 video_stream_encoder_->SetSource(&source,
3821 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003822 timestamp_ms += kFrameIntervalMs;
3823 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003824 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003825 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003826 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3828 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3829
3830 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003831 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003832 timestamp_ms += kFrameIntervalMs;
3833 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003834 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003835 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3836 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3837 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3838 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3839
3840 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003841 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003842 timestamp_ms += kFrameIntervalMs;
3843 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003844 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003845 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3846 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3847 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3848 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3849
3850 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003851 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003852 timestamp_ms += kFrameIntervalMs;
3853 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003854 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003855 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3856 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3857 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3858 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3859
3860 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003861 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003862 timestamp_ms += kFrameIntervalMs;
3863 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003864 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003865 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3866 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3867 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3868 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3869
3870 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003871 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003872 timestamp_ms += kFrameIntervalMs;
3873 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003874 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003875 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3876 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3878 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3879
3880 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003881 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003882 timestamp_ms += kFrameIntervalMs;
3883 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003884 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003885 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3886 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3887 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3888 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3889
3890 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003891 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003892 timestamp_ms += kFrameIntervalMs;
3893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003894 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003895 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3896 rtc::VideoSinkWants last_wants = source.sink_wants();
3897 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3898 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3899 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3900
3901 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003902 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003903 timestamp_ms += kFrameIntervalMs;
3904 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003905 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003906 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3907 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3908 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3909 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3910
3911 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003912 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003913 timestamp_ms += kFrameIntervalMs;
3914 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003915 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003916 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3917 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3918 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3919 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3920
3921 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003922 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003923 timestamp_ms += kFrameIntervalMs;
3924 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003925 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003926 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3927 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3928 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3929 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3930
3931 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003932 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003933 timestamp_ms += kFrameIntervalMs;
3934 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003935 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003936 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3937 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3938 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3939 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3940
3941 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003942 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003943 timestamp_ms += kFrameIntervalMs;
3944 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003945 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003946 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3947 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3948 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3949 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3950
3951 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003952 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003953 timestamp_ms += kFrameIntervalMs;
3954 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003955 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003956 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3957 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3958 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3959 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3960
3961 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003962 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003963 timestamp_ms += kFrameIntervalMs;
3964 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003965 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003966 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3967 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3968 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3969 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3970
Åsa Persson30ab0152019-08-27 12:22:33 +02003971 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003972 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003973 timestamp_ms += kFrameIntervalMs;
3974 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003975 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003976 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003977 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003978 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3979 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3980 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3981
3982 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003983 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003984 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003985 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3986
mflodmancc3d4422017-08-03 08:27:51 -07003987 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003988}
3989
mflodmancc3d4422017-08-03 08:27:51 -07003990TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003991 const int kWidth = 1280;
3992 const int kHeight = 720;
3993 const int64_t kFrameIntervalMs = 150;
3994 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003995 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003996 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3997 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003998
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003999 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004000 AdaptingFrameForwarder source;
4001 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004002 video_stream_encoder_->SetSource(&source,
4003 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004004 timestamp_ms += kFrameIntervalMs;
4005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004006 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004007 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004008 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4009 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4010 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4011 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4012 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4013 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4014
4015 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004016 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004017 timestamp_ms += kFrameIntervalMs;
4018 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004019 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004020 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4021 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4023 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4024 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4025 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4026 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4027
4028 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004029 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004030 timestamp_ms += kFrameIntervalMs;
4031 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004032 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004033 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4035 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4036 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4037 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4038 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4039 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4040
4041 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004042 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004043 timestamp_ms += kFrameIntervalMs;
4044 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004045 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004046 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4047 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4048 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4049 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4050 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4051 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4052 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4053
4054 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004055 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004056 timestamp_ms += kFrameIntervalMs;
4057 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004058 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004059 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4060 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4061 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4062 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4063 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4064 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4065 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4066
4067 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004068 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004069 timestamp_ms += kFrameIntervalMs;
4070 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004071 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004072 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4073 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4074 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4076 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4077 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4078 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4079
4080 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004081 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004082 timestamp_ms += kFrameIntervalMs;
4083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004084 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004085 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004086 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004087 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4088 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4089 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4090 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4091 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4092 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4093
4094 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004095 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004096 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004097 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4098 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4099
mflodmancc3d4422017-08-03 08:27:51 -07004100 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004101}
4102
mflodmancc3d4422017-08-03 08:27:51 -07004103TEST_F(VideoStreamEncoderTest,
4104 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004105 const int kWidth = 640;
4106 const int kHeight = 360;
4107 const int kFpsLimit = 15;
4108 const int64_t kFrameIntervalMs = 150;
4109 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004110 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004111 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4112 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004113
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004114 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004115 AdaptingFrameForwarder source;
4116 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004117 video_stream_encoder_->SetSource(&source,
4118 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004119 timestamp_ms += kFrameIntervalMs;
4120 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004121 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004122 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004123 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4124 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4125 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4126 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4127 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4128 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4129
4130 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004131 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004132 timestamp_ms += kFrameIntervalMs;
4133 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004134 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004135 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4136 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4137 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4138 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4139 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4140 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4141 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4142
4143 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004144 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004145 timestamp_ms += kFrameIntervalMs;
4146 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004147 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004148 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4149 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4150 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4151 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4152 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4153 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4154 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4155
4156 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004157 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004158 timestamp_ms += kFrameIntervalMs;
4159 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004160 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004161 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4162 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4163 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4164 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4165 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4166 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4167 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4168
4169 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004170 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004171 timestamp_ms += kFrameIntervalMs;
4172 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004173 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004174 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004175 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4176 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4177 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4178 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4179 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4180 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4181
4182 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004183 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004184 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004185 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4186 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4187
mflodmancc3d4422017-08-03 08:27:51 -07004188 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004189}
4190
mflodmancc3d4422017-08-03 08:27:51 -07004191TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004192 const int kFrameWidth = 1920;
4193 const int kFrameHeight = 1080;
4194 // 3/4 of 1920.
4195 const int kAdaptedFrameWidth = 1440;
4196 // 3/4 of 1080 rounded down to multiple of 4.
4197 const int kAdaptedFrameHeight = 808;
4198 const int kFramerate = 24;
4199
Erik Språng4c6ca302019-04-08 15:14:01 +02004200 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004201 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4202 DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004203 // Trigger reconfigure encoder (without resetting the entire instance).
4204 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004205 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004206 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4207 video_encoder_config.number_of_streams = 1;
4208 video_encoder_config.video_stream_factory =
4209 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004210 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004211 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004212 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004213
4214 video_source_.set_adaptation_enabled(true);
4215
4216 video_source_.IncomingCapturedFrame(
4217 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004218 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004219
4220 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004221 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004222 video_source_.IncomingCapturedFrame(
4223 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004224 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004225
mflodmancc3d4422017-08-03 08:27:51 -07004226 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004227}
4228
mflodmancc3d4422017-08-03 08:27:51 -07004229TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004230 const int kFrameWidth = 1280;
4231 const int kFrameHeight = 720;
4232 const int kLowFps = 2;
4233 const int kHighFps = 30;
4234
Erik Språng4c6ca302019-04-08 15:14:01 +02004235 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004236 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4237 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004238
4239 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4240 max_framerate_ = kLowFps;
4241
4242 // Insert 2 seconds of 2fps video.
4243 for (int i = 0; i < kLowFps * 2; ++i) {
4244 video_source_.IncomingCapturedFrame(
4245 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4246 WaitForEncodedFrame(timestamp_ms);
4247 timestamp_ms += 1000 / kLowFps;
4248 }
4249
4250 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004251 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004252 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4253 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004254 video_source_.IncomingCapturedFrame(
4255 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4256 WaitForEncodedFrame(timestamp_ms);
4257 timestamp_ms += 1000 / kLowFps;
4258
4259 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4260
4261 // Insert 30fps frames for just a little more than the forced update period.
4262 const int kVcmTimerIntervalFrames =
4263 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
4264 const int kFrameIntervalMs = 1000 / kHighFps;
4265 max_framerate_ = kHighFps;
4266 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4267 video_source_.IncomingCapturedFrame(
4268 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4269 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4270 // be dropped if the encoder hans't been updated with the new higher target
4271 // framerate yet, causing it to overshoot the target bitrate and then
4272 // suffering the wrath of the media optimizer.
4273 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4274 timestamp_ms += kFrameIntervalMs;
4275 }
4276
4277 // Don expect correct measurement just yet, but it should be higher than
4278 // before.
4279 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4280
mflodmancc3d4422017-08-03 08:27:51 -07004281 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004282}
4283
mflodmancc3d4422017-08-03 08:27:51 -07004284TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004285 const int kFrameWidth = 1280;
4286 const int kFrameHeight = 720;
4287 const int kTargetBitrateBps = 1000000;
4288
4289 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004290 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004291 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004292 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4293 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004294 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004295
4296 // Insert a first video frame, causes another bitrate update.
4297 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4298 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4299 video_source_.IncomingCapturedFrame(
4300 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4301 WaitForEncodedFrame(timestamp_ms);
4302
4303 // Next, simulate video suspension due to pacer queue overrun.
Florent Castellia8336d32019-09-09 13:36:55 +02004304 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
4305 DataRate::bps(0), 0, 1);
sprang4847ae62017-06-27 07:06:52 -07004306
4307 // Skip ahead until a new periodic parameter update should have occured.
4308 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02004309 fake_clock_.AdvanceTime(
4310 TimeDelta::ms(vcm::VCMProcessTimer::kDefaultProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004311
4312 // Bitrate observer should not be called.
4313 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4314 video_source_.IncomingCapturedFrame(
4315 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4316 ExpectDroppedFrame();
4317
mflodmancc3d4422017-08-03 08:27:51 -07004318 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004319}
ilnik6b826ef2017-06-16 06:53:48 -07004320
Niels Möller4db138e2018-04-19 09:04:13 +02004321TEST_F(VideoStreamEncoderTest,
4322 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4323 const int kFrameWidth = 1280;
4324 const int kFrameHeight = 720;
4325 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004326 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004327 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4328 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004329 video_source_.IncomingCapturedFrame(
4330 CreateFrame(1, kFrameWidth, kFrameHeight));
4331 WaitForEncodedFrame(1);
4332 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4333 .low_encode_usage_threshold_percent,
4334 default_options.low_encode_usage_threshold_percent);
4335 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4336 .high_encode_usage_threshold_percent,
4337 default_options.high_encode_usage_threshold_percent);
4338 video_stream_encoder_->Stop();
4339}
4340
4341TEST_F(VideoStreamEncoderTest,
4342 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4343 const int kFrameWidth = 1280;
4344 const int kFrameHeight = 720;
4345 CpuOveruseOptions hardware_options;
4346 hardware_options.low_encode_usage_threshold_percent = 150;
4347 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004348 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004349
Erik Språng4c6ca302019-04-08 15:14:01 +02004350 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004351 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4352 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004353 video_source_.IncomingCapturedFrame(
4354 CreateFrame(1, kFrameWidth, kFrameHeight));
4355 WaitForEncodedFrame(1);
4356 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4357 .low_encode_usage_threshold_percent,
4358 hardware_options.low_encode_usage_threshold_percent);
4359 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4360 .high_encode_usage_threshold_percent,
4361 hardware_options.high_encode_usage_threshold_percent);
4362 video_stream_encoder_->Stop();
4363}
4364
Niels Möller6bb5ab92019-01-11 11:11:10 +01004365TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4366 const int kFrameWidth = 320;
4367 const int kFrameHeight = 240;
4368 const int kFps = 30;
4369 const int kTargetBitrateBps = 120000;
4370 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4371
Erik Språng4c6ca302019-04-08 15:14:01 +02004372 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004373 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4374 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004375
4376 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4377 max_framerate_ = kFps;
4378
4379 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4380 fake_encoder_.SimulateOvershoot(1.0);
4381 int num_dropped = 0;
4382 for (int i = 0; i < kNumFramesInRun; ++i) {
4383 video_source_.IncomingCapturedFrame(
4384 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4385 // Wait up to two frame durations for a frame to arrive.
4386 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4387 ++num_dropped;
4388 }
4389 timestamp_ms += 1000 / kFps;
4390 }
4391
Erik Språnga8d48ab2019-02-08 14:17:40 +01004392 // Framerate should be measured to be near the expected target rate.
4393 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4394
4395 // Frame drops should be within 5% of expected 0%.
4396 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004397
4398 // Make encoder produce frames at double the expected bitrate during 3 seconds
4399 // of video, verify number of drops. Rate needs to be slightly changed in
4400 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004401 double overshoot_factor = 2.0;
4402 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4403 // With bitrate adjuster, when need to overshoot even more to trigger
4404 // frame dropping.
4405 overshoot_factor *= 2;
4406 }
4407 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004408 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004409 DataRate::bps(kTargetBitrateBps + 1000),
Florent Castellia8336d32019-09-09 13:36:55 +02004410 DataRate::bps(kTargetBitrateBps + 1000),
Erik Språng4c6ca302019-04-08 15:14:01 +02004411 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004412 num_dropped = 0;
4413 for (int i = 0; i < kNumFramesInRun; ++i) {
4414 video_source_.IncomingCapturedFrame(
4415 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4416 // Wait up to two frame durations for a frame to arrive.
4417 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4418 ++num_dropped;
4419 }
4420 timestamp_ms += 1000 / kFps;
4421 }
4422
Erik Språng4c6ca302019-04-08 15:14:01 +02004423 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004424 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4425 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004426
4427 // Target framerate should be still be near the expected target, despite
4428 // the frame drops.
4429 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4430
4431 // Frame drops should be within 5% of expected 50%.
4432 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004433
4434 video_stream_encoder_->Stop();
4435}
4436
4437TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4438 const int kFrameWidth = 320;
4439 const int kFrameHeight = 240;
4440 const int kActualInputFps = 24;
4441 const int kTargetBitrateBps = 120000;
4442
4443 ASSERT_GT(max_framerate_, kActualInputFps);
4444
4445 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4446 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004447 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004448 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4449 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004450
4451 // Insert 3 seconds of video, with an input fps lower than configured max.
4452 for (int i = 0; i < kActualInputFps * 3; ++i) {
4453 video_source_.IncomingCapturedFrame(
4454 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4455 // Wait up to two frame durations for a frame to arrive.
4456 WaitForEncodedFrame(timestamp_ms);
4457 timestamp_ms += 1000 / kActualInputFps;
4458 }
4459
4460 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4461
4462 video_stream_encoder_->Stop();
4463}
4464
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004465TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4466 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004467 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004468 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4469 DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004470
4471 fake_encoder_.BlockNextEncode();
4472 video_source_.IncomingCapturedFrame(
4473 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4474 WaitForEncodedFrame(1);
4475 // On the very first frame full update should be forced.
4476 rect = fake_encoder_.GetLastUpdateRect();
4477 EXPECT_EQ(rect.offset_x, 0);
4478 EXPECT_EQ(rect.offset_y, 0);
4479 EXPECT_EQ(rect.height, codec_height_);
4480 EXPECT_EQ(rect.width, codec_width_);
4481 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4482 // call to ContinueEncode.
4483 video_source_.IncomingCapturedFrame(
4484 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4485 ExpectDroppedFrame();
4486 video_source_.IncomingCapturedFrame(
4487 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4488 ExpectDroppedFrame();
4489 fake_encoder_.ContinueEncode();
4490 WaitForEncodedFrame(3);
4491 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4492 rect = fake_encoder_.GetLastUpdateRect();
4493 EXPECT_EQ(rect.offset_x, 1);
4494 EXPECT_EQ(rect.offset_y, 0);
4495 EXPECT_EQ(rect.width, 10);
4496 EXPECT_EQ(rect.height, 1);
4497
4498 video_source_.IncomingCapturedFrame(
4499 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4500 WaitForEncodedFrame(4);
4501 // Previous frame was encoded, so no accumulation should happen.
4502 rect = fake_encoder_.GetLastUpdateRect();
4503 EXPECT_EQ(rect.offset_x, 0);
4504 EXPECT_EQ(rect.offset_y, 0);
4505 EXPECT_EQ(rect.width, 1);
4506 EXPECT_EQ(rect.height, 1);
4507
4508 video_stream_encoder_->Stop();
4509}
4510
Erik Språngd7329ca2019-02-21 21:19:53 +01004511TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004512 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004513 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4514 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004515
4516 // First frame is always keyframe.
4517 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4518 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004519 EXPECT_THAT(
4520 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004521 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004522
4523 // Insert delta frame.
4524 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4525 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01004526 EXPECT_THAT(
4527 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004528 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004529
4530 // Request next frame be a key-frame.
4531 video_stream_encoder_->SendKeyFrame();
4532 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4533 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01004534 EXPECT_THAT(
4535 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004536 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004537
4538 video_stream_encoder_->Stop();
4539}
4540
4541TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4542 // Setup simulcast with three streams.
4543 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01004544 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004545 DataRate::bps(kSimulcastTargetBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02004546 DataRate::bps(kSimulcastTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02004547 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004548 // Wait for all three layers before triggering event.
4549 sink_.SetNumExpectedLayers(3);
4550
4551 // First frame is always keyframe.
4552 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4553 WaitForEncodedFrame(1);
4554 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004555 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4556 VideoFrameType::kVideoFrameKey,
4557 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004558
4559 // Insert delta frame.
4560 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4561 WaitForEncodedFrame(2);
4562 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004563 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4564 VideoFrameType::kVideoFrameDelta,
4565 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004566
4567 // Request next frame be a key-frame.
4568 // Only first stream is configured to produce key-frame.
4569 video_stream_encoder_->SendKeyFrame();
4570 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4571 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02004572
4573 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4574 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01004575 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004576 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02004577 VideoFrameType::kVideoFrameKey,
4578 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004579
4580 video_stream_encoder_->Stop();
4581}
4582
4583TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
4584 // Configure internal source factory and setup test again.
4585 encoder_factory_.SetHasInternalSource(true);
4586 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004587 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004588 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4589 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004590
4591 // Call encoder directly, simulating internal source where encoded frame
4592 // callback in VideoStreamEncoder is called despite no OnFrame().
4593 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
4594 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004595 EXPECT_THAT(
4596 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004597 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004598
Niels Möller8f7ce222019-03-21 15:43:58 +01004599 const std::vector<VideoFrameType> kDeltaFrame = {
4600 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01004601 // Need to set timestamp manually since manually for injected frame.
4602 VideoFrame frame = CreateFrame(101, nullptr);
4603 frame.set_timestamp(101);
4604 fake_encoder_.InjectFrame(frame, false);
4605 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004606 EXPECT_THAT(
4607 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004608 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004609
4610 // Request key-frame. The forces a dummy frame down into the encoder.
4611 fake_encoder_.ExpectNullFrame();
4612 video_stream_encoder_->SendKeyFrame();
4613 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004614 EXPECT_THAT(
4615 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004616 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004617
4618 video_stream_encoder_->Stop();
4619}
Erik Språngb7cb7b52019-02-26 15:52:33 +01004620
4621TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
4622 // Configure internal source factory and setup test again.
4623 encoder_factory_.SetHasInternalSource(true);
4624 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004625 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004626 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4627 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01004628
4629 int64_t timestamp = 1;
4630 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02004631 image.SetEncodedData(
4632 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01004633 image.capture_time_ms_ = ++timestamp;
4634 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
4635 const int64_t kEncodeFinishDelayMs = 10;
4636 image.timing_.encode_start_ms = timestamp;
4637 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
4638 fake_encoder_.InjectEncodedImage(image);
4639 // Wait for frame without incrementing clock.
4640 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4641 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
4642 // capture timestamp should be kEncodeFinishDelayMs in the past.
4643 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
4644 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
4645 kEncodeFinishDelayMs);
4646
4647 video_stream_encoder_->Stop();
4648}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02004649
4650TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
4651 // Configure internal source factory and setup test again.
4652 encoder_factory_.SetHasInternalSource(true);
4653 ResetEncoder("H264", 1, 1, 1, false);
4654
4655 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
4656 image._frameType = VideoFrameType::kVideoFrameKey;
4657
4658 CodecSpecificInfo codec_specific_info;
4659 codec_specific_info.codecType = kVideoCodecH264;
4660
4661 RTPFragmentationHeader fragmentation;
4662 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4663 fragmentation.fragmentationOffset[0] = 4;
4664 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
4665
4666 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4667 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4668
4669 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4670 testing::ElementsAreArray(optimal_sps));
4671 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4672 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4673 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4674 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4675
4676 video_stream_encoder_->Stop();
4677}
4678
4679TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
4680 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
4681 0x00, 0x00, 0x03, 0x03, 0xF4,
4682 0x05, 0x03, 0xC7, 0xC0};
4683
4684 // Configure internal source factory and setup test again.
4685 encoder_factory_.SetHasInternalSource(true);
4686 ResetEncoder("H264", 1, 1, 1, false);
4687
4688 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
4689 image._frameType = VideoFrameType::kVideoFrameKey;
4690
4691 CodecSpecificInfo codec_specific_info;
4692 codec_specific_info.codecType = kVideoCodecH264;
4693
4694 RTPFragmentationHeader fragmentation;
4695 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4696 fragmentation.fragmentationOffset[0] = 4;
4697 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
4698
4699 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4700 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4701
4702 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4703 testing::ElementsAreArray(optimal_sps));
4704 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4705 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4706 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4707 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4708
4709 video_stream_encoder_->Stop();
4710}
4711
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02004712TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
4713 const int kFrameWidth = 1280;
4714 const int kFrameHeight = 720;
4715 const int kTargetBitrateBps = 300000; // To low for HD resolution.
4716
4717 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004718 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4719 DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02004720 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4721
4722 // Insert a first video frame. It should be dropped because of downscale in
4723 // resolution.
4724 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4725 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4726 frame.set_rotation(kVideoRotation_270);
4727 video_source_.IncomingCapturedFrame(frame);
4728
4729 ExpectDroppedFrame();
4730
4731 // Second frame is downscaled.
4732 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4733 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4734 frame.set_rotation(kVideoRotation_90);
4735 video_source_.IncomingCapturedFrame(frame);
4736
4737 WaitForEncodedFrame(timestamp_ms);
4738 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
4739
4740 // Insert another frame, also downscaled.
4741 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4742 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4743 frame.set_rotation(kVideoRotation_180);
4744 video_source_.IncomingCapturedFrame(frame);
4745
4746 WaitForEncodedFrame(timestamp_ms);
4747 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
4748
4749 video_stream_encoder_->Stop();
4750}
4751
Erik Språng5056af02019-09-02 15:53:11 +02004752TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
4753 const int kFrameWidth = 320;
4754 const int kFrameHeight = 180;
4755
4756 // Initial rate.
4757 video_stream_encoder_->OnBitrateUpdated(
4758 /*target_bitrate=*/DataRate::kbps(300),
Florent Castellia8336d32019-09-09 13:36:55 +02004759 /*stable_target_bitrate=*/DataRate::kbps(300),
Erik Språng5056af02019-09-02 15:53:11 +02004760 /*link_allocation=*/DataRate::kbps(300),
4761 /*fraction_lost=*/0,
4762 /*rtt_ms=*/0);
4763
4764 // Insert a first video frame so that encoder gets configured.
4765 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4766 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4767 frame.set_rotation(kVideoRotation_270);
4768 video_source_.IncomingCapturedFrame(frame);
4769 WaitForEncodedFrame(timestamp_ms);
4770
4771 // Set a target rate below the minimum allowed by the codec settings.
4772 VideoCodec codec_config = fake_encoder_.codec_config();
4773 DataRate min_rate = DataRate::kbps(codec_config.minBitrate);
4774 DataRate target_rate = min_rate - DataRate::kbps(1);
4775 video_stream_encoder_->OnBitrateUpdated(
4776 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02004777 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02004778 /*link_allocation=*/target_rate,
4779 /*fraction_lost=*/0,
4780 /*rtt_ms=*/0);
4781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4782
4783 // Target bitrate and bandwidth allocation should both be capped at min_rate.
4784 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
4785 ASSERT_TRUE(rate_settings.has_value());
4786 DataRate allocation_sum = DataRate::bps(rate_settings->bitrate.get_sum_bps());
4787 EXPECT_EQ(min_rate, allocation_sum);
4788 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
4789
4790 video_stream_encoder_->Stop();
4791}
4792
perkj26091b12016-09-01 01:17:40 -07004793} // namespace webrtc