blob: 1ad4cbffd5c23bdccc241ffd8e579c6d45925b4b [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080021#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010023#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020024#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020025#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010026#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020027#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070028#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080029#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020030#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010032#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010036#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020037#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "system_wrappers/include/sleep.h"
39#include "test/encoder_settings.h"
40#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020041#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "test/frame_generator.h"
43#include "test/gmock.h"
44#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020045#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070047
48namespace webrtc {
49
sprangb1ca0732017-02-01 08:38:12 -080050using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080051using ::testing::_;
kthelgason876222f2016-11-29 01:44:11 -080052
perkj803d97f2016-11-01 11:45:46 -070053namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020054const int kMinPixelsPerFrame = 320 * 180;
55const int kMinFramerateFps = 2;
56const int kMinBalancedFramerateFps = 7;
57const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080058const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010059const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020060const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010061const uint32_t kSimulcastTargetBitrateBps = 3150000;
62const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080063const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070064const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080066
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020067uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
68 0x00, 0x00, 0x03, 0x03, 0xF4,
69 0x05, 0x03, 0xC7, 0xE0, 0x1B,
70 0x41, 0x10, 0x8D, 0x00};
71
perkj803d97f2016-11-01 11:45:46 -070072class TestBuffer : public webrtc::I420Buffer {
73 public:
74 TestBuffer(rtc::Event* event, int width, int height)
75 : I420Buffer(width, height), event_(event) {}
76
77 private:
78 friend class rtc::RefCountedObject<TestBuffer>;
79 ~TestBuffer() override {
80 if (event_)
81 event_->Set();
82 }
83 rtc::Event* const event_;
84};
85
Noah Richards51db4212019-06-12 06:59:12 -070086// A fake native buffer that can't be converted to I420.
87class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
88 public:
89 FakeNativeBuffer(rtc::Event* event, int width, int height)
90 : event_(event), width_(width), height_(height) {}
91 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
92 int width() const override { return width_; }
93 int height() const override { return height_; }
94 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
95 return nullptr;
96 }
97
98 private:
99 friend class rtc::RefCountedObject<FakeNativeBuffer>;
100 ~FakeNativeBuffer() override {
101 if (event_)
102 event_->Set();
103 }
104 rtc::Event* const event_;
105 const int width_;
106 const int height_;
107};
108
Niels Möller7dc26b72017-12-06 10:27:48 +0100109class CpuOveruseDetectorProxy : public OveruseFrameDetector {
110 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200111 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
112 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100113 last_target_framerate_fps_(-1) {}
114 virtual ~CpuOveruseDetectorProxy() {}
115
116 void OnTargetFramerateUpdated(int framerate_fps) override {
117 rtc::CritScope cs(&lock_);
118 last_target_framerate_fps_ = framerate_fps;
119 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
120 }
121
122 int GetLastTargetFramerate() {
123 rtc::CritScope cs(&lock_);
124 return last_target_framerate_fps_;
125 }
126
Niels Möller4db138e2018-04-19 09:04:13 +0200127 CpuOveruseOptions GetOptions() { return options_; }
128
Niels Möller7dc26b72017-12-06 10:27:48 +0100129 private:
130 rtc::CriticalSection lock_;
131 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
132};
133
mflodmancc3d4422017-08-03 08:27:51 -0700134class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700135 public:
Niels Möller213618e2018-07-24 09:29:58 +0200136 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200137 const VideoStreamEncoderSettings& settings,
138 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100139 : VideoStreamEncoder(Clock::GetRealTimeClock(),
140 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200141 stats_proxy,
142 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200143 std::unique_ptr<OveruseFrameDetector>(
144 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100145 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200146 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700147
sprangb1ca0732017-02-01 08:38:12 -0800148 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100149 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800150 encoder_queue()->PostTask([this, &event, reason, down] {
Åsa Perssonf5e5d252019-08-16 17:24:59 +0200151 if (down)
152 AdaptDown(reason);
153 else
154 AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700155 event.Set();
156 });
perkj070ba852017-02-16 15:46:27 -0800157 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700158 }
159
kthelgason2fc52542017-03-03 00:24:41 -0800160 // This is used as a synchronisation mechanism, to make sure that the
161 // encoder queue is not blocked before we start sending it frames.
162 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100163 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200164 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800165 ASSERT_TRUE(event.Wait(5000));
166 }
167
sprangb1ca0732017-02-01 08:38:12 -0800168 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800169
sprangb1ca0732017-02-01 08:38:12 -0800170 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800171
sprangb1ca0732017-02-01 08:38:12 -0800172 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800173
sprangb1ca0732017-02-01 08:38:12 -0800174 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700175
Niels Möller7dc26b72017-12-06 10:27:48 +0100176 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700177};
178
asapersson5f7226f2016-11-25 04:37:00 -0800179class VideoStreamFactory
180 : public VideoEncoderConfig::VideoStreamFactoryInterface {
181 public:
sprangfda496a2017-06-15 04:21:07 -0700182 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
183 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800184 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700185 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800186 }
187
188 private:
189 std::vector<VideoStream> CreateEncoderStreams(
190 int width,
191 int height,
192 const VideoEncoderConfig& encoder_config) override {
193 std::vector<VideoStream> streams =
194 test::CreateVideoStreams(width, height, encoder_config);
195 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100196 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700197 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800198 }
199 return streams;
200 }
sprangfda496a2017-06-15 04:21:07 -0700201
asapersson5f7226f2016-11-25 04:37:00 -0800202 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700203 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800204};
205
Noah Richards51db4212019-06-12 06:59:12 -0700206// Simulates simulcast behavior and makes highest stream resolutions divisible
207// by 4.
208class CroppingVideoStreamFactory
209 : public VideoEncoderConfig::VideoStreamFactoryInterface {
210 public:
211 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
212 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
213 EXPECT_GT(num_temporal_layers, 0u);
214 EXPECT_GT(framerate, 0);
215 }
216
217 private:
218 std::vector<VideoStream> CreateEncoderStreams(
219 int width,
220 int height,
221 const VideoEncoderConfig& encoder_config) override {
222 std::vector<VideoStream> streams = test::CreateVideoStreams(
223 width - width % 4, height - height % 4, encoder_config);
224 for (VideoStream& stream : streams) {
225 stream.num_temporal_layers = num_temporal_layers_;
226 stream.max_framerate = framerate_;
227 }
228 return streams;
229 }
230
231 const size_t num_temporal_layers_;
232 const int framerate_;
233};
234
sprangb1ca0732017-02-01 08:38:12 -0800235class AdaptingFrameForwarder : public test::FrameForwarder {
236 public:
237 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700238 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800239
240 void set_adaptation_enabled(bool enabled) {
241 rtc::CritScope cs(&crit_);
242 adaptation_enabled_ = enabled;
243 }
244
asaperssonfab67072017-04-04 05:51:49 -0700245 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800246 rtc::CritScope cs(&crit_);
247 return adaptation_enabled_;
248 }
249
asapersson09f05612017-05-15 23:40:18 -0700250 rtc::VideoSinkWants last_wants() const {
251 rtc::CritScope cs(&crit_);
252 return last_wants_;
253 }
254
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200255 absl::optional<int> last_sent_width() const { return last_width_; }
256 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800257
sprangb1ca0732017-02-01 08:38:12 -0800258 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
259 int cropped_width = 0;
260 int cropped_height = 0;
261 int out_width = 0;
262 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700263 if (adaption_enabled()) {
264 if (adapter_.AdaptFrameResolution(
265 video_frame.width(), video_frame.height(),
266 video_frame.timestamp_us() * 1000, &cropped_width,
267 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100268 VideoFrame adapted_frame =
269 VideoFrame::Builder()
270 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
271 nullptr, out_width, out_height))
272 .set_timestamp_rtp(99)
273 .set_timestamp_ms(99)
274 .set_rotation(kVideoRotation_0)
275 .build();
sprangc5d62e22017-04-02 23:53:04 -0700276 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
277 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800278 last_width_.emplace(adapted_frame.width());
279 last_height_.emplace(adapted_frame.height());
280 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200281 last_width_ = absl::nullopt;
282 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700283 }
sprangb1ca0732017-02-01 08:38:12 -0800284 } else {
285 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800286 last_width_.emplace(video_frame.width());
287 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800288 }
289 }
290
291 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
292 const rtc::VideoSinkWants& wants) override {
293 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700294 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700295 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
296 wants.max_pixel_count,
297 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800298 test::FrameForwarder::AddOrUpdateSink(sink, wants);
299 }
sprangb1ca0732017-02-01 08:38:12 -0800300 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700301 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
302 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200303 absl::optional<int> last_width_;
304 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800305};
sprangc5d62e22017-04-02 23:53:04 -0700306
Niels Möller213618e2018-07-24 09:29:58 +0200307// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700308class MockableSendStatisticsProxy : public SendStatisticsProxy {
309 public:
310 MockableSendStatisticsProxy(Clock* clock,
311 const VideoSendStream::Config& config,
312 VideoEncoderConfig::ContentType content_type)
313 : SendStatisticsProxy(clock, config, content_type) {}
314
315 VideoSendStream::Stats GetStats() override {
316 rtc::CritScope cs(&lock_);
317 if (mock_stats_)
318 return *mock_stats_;
319 return SendStatisticsProxy::GetStats();
320 }
321
Niels Möller213618e2018-07-24 09:29:58 +0200322 int GetInputFrameRate() const override {
323 rtc::CritScope cs(&lock_);
324 if (mock_stats_)
325 return mock_stats_->input_frame_rate;
326 return SendStatisticsProxy::GetInputFrameRate();
327 }
sprangc5d62e22017-04-02 23:53:04 -0700328 void SetMockStats(const VideoSendStream::Stats& stats) {
329 rtc::CritScope cs(&lock_);
330 mock_stats_.emplace(stats);
331 }
332
333 void ResetMockStats() {
334 rtc::CritScope cs(&lock_);
335 mock_stats_.reset();
336 }
337
338 private:
339 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200340 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700341};
342
sprang4847ae62017-06-27 07:06:52 -0700343class MockBitrateObserver : public VideoBitrateAllocationObserver {
344 public:
Erik Språng566124a2018-04-23 12:32:22 +0200345 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700346};
347
perkj803d97f2016-11-01 11:45:46 -0700348} // namespace
349
mflodmancc3d4422017-08-03 08:27:51 -0700350class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700351 public:
352 static const int kDefaultTimeoutMs = 30 * 1000;
353
mflodmancc3d4422017-08-03 08:27:51 -0700354 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700355 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700356 codec_width_(320),
357 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200358 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200359 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700360 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200361 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700362 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700363 Clock::GetRealTimeClock(),
364 video_send_config_,
365 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700366 sink_(&fake_encoder_) {}
367
368 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700369 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700370 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200371 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800372 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200373 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200374 video_send_config_.rtp.payload_name = "FAKE";
375 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700376
Per512ecb32016-09-23 15:52:06 +0200377 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200378 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700379 video_encoder_config.video_stream_factory =
380 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100381 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700382
383 // Framerate limit is specified by the VideoStreamFactory.
384 std::vector<VideoStream> streams =
385 video_encoder_config.video_stream_factory->CreateEncoderStreams(
386 codec_width_, codec_height_, video_encoder_config);
387 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200388 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700389
Niels Möllerf1338562018-04-26 09:51:47 +0200390 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800391 }
392
Niels Möllerf1338562018-04-26 09:51:47 +0200393 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700394 if (video_stream_encoder_)
395 video_stream_encoder_->Stop();
396 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200397 stats_proxy_.get(), video_send_config_.encoder_settings,
398 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700399 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
400 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700401 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700402 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
403 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200404 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700405 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800406 }
407
408 void ResetEncoder(const std::string& payload_name,
409 size_t num_streams,
410 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700411 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700412 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200413 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800414
415 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200416 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800417 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100418 video_encoder_config.max_bitrate_bps =
419 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800420 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700421 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
422 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700423 video_encoder_config.content_type =
424 screenshare ? VideoEncoderConfig::ContentType::kScreen
425 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700426 if (payload_name == "VP9") {
427 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
428 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
429 video_encoder_config.encoder_specific_settings =
430 new rtc::RefCountedObject<
431 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
432 }
Niels Möllerf1338562018-04-26 09:51:47 +0200433 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700434 }
435
sprang57c2fff2017-01-16 06:24:02 -0800436 VideoFrame CreateFrame(int64_t ntp_time_ms,
437 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100438 VideoFrame frame =
439 VideoFrame::Builder()
440 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
441 destruction_event, codec_width_, codec_height_))
442 .set_timestamp_rtp(99)
443 .set_timestamp_ms(99)
444 .set_rotation(kVideoRotation_0)
445 .build();
sprang57c2fff2017-01-16 06:24:02 -0800446 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700447 return frame;
448 }
449
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100450 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
451 rtc::Event* destruction_event,
452 int offset_x) const {
453 VideoFrame frame =
454 VideoFrame::Builder()
455 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
456 destruction_event, codec_width_, codec_height_))
457 .set_timestamp_rtp(99)
458 .set_timestamp_ms(99)
459 .set_rotation(kVideoRotation_0)
460 .set_update_rect({offset_x, 0, 1, 1})
461 .build();
462 frame.set_ntp_time_ms(ntp_time_ms);
463 return frame;
464 }
465
sprang57c2fff2017-01-16 06:24:02 -0800466 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100467 VideoFrame frame =
468 VideoFrame::Builder()
469 .set_video_frame_buffer(
470 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
471 .set_timestamp_rtp(99)
472 .set_timestamp_ms(99)
473 .set_rotation(kVideoRotation_0)
474 .build();
sprang57c2fff2017-01-16 06:24:02 -0800475 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700476 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700477 return frame;
478 }
479
Noah Richards51db4212019-06-12 06:59:12 -0700480 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
481 rtc::Event* destruction_event,
482 int width,
483 int height) const {
484 VideoFrame frame =
485 VideoFrame::Builder()
486 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
487 destruction_event, width, height))
488 .set_timestamp_rtp(99)
489 .set_timestamp_ms(99)
490 .set_rotation(kVideoRotation_0)
491 .build();
492 frame.set_ntp_time_ms(ntp_time_ms);
493 return frame;
494 }
495
496 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
497 rtc::Event* destruction_event) const {
498 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
499 codec_height_);
500 }
501
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100502 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
503 MockBitrateObserver bitrate_observer;
504 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
505
506 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
507 .Times(1);
508 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +0200509 DataRate::bps(kTargetBitrateBps), 0,
510 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100511
512 video_source_.IncomingCapturedFrame(
513 CreateFrame(1, codec_width_, codec_height_));
514 WaitForEncodedFrame(1);
515 }
516
asapersson02465b82017-04-10 01:12:52 -0700517 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700518 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700519 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
520 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700521 }
522
asapersson09f05612017-05-15 23:40:18 -0700523 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
524 const rtc::VideoSinkWants& wants2) {
525 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
526 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
527 }
528
Åsa Persson8c1bf952018-09-13 10:42:19 +0200529 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
530 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
531 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
532 EXPECT_FALSE(wants.target_pixel_count);
533 }
534
asapersson09f05612017-05-15 23:40:18 -0700535 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
536 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200537 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700538 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
539 EXPECT_GT(wants1.max_pixel_count, 0);
540 }
541
542 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
543 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200544 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700545 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
546 }
547
asaperssonf7e294d2017-06-13 23:25:22 -0700548 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
549 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200550 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700551 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
552 }
553
554 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
555 const rtc::VideoSinkWants& wants2) {
556 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
557 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
558 }
559
560 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
561 const rtc::VideoSinkWants& wants2) {
562 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
563 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
564 }
565
566 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
567 const rtc::VideoSinkWants& wants2) {
568 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
569 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
570 EXPECT_GT(wants1.max_pixel_count, 0);
571 }
572
573 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
574 const rtc::VideoSinkWants& wants2) {
575 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
576 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
577 }
578
asapersson09f05612017-05-15 23:40:18 -0700579 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
580 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200581 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700582 EXPECT_LT(wants.max_pixel_count, pixel_count);
583 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700584 }
585
586 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
587 EXPECT_LT(wants.max_framerate_fps, fps);
588 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
589 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700590 }
591
asaperssonf7e294d2017-06-13 23:25:22 -0700592 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
593 int expected_fps) {
594 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
595 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
596 EXPECT_FALSE(wants.target_pixel_count);
597 }
598
Jonathan Yubc771b72017-12-08 17:04:29 -0800599 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
600 int last_frame_pixels) {
601 // Balanced mode should always scale FPS to the desired range before
602 // attempting to scale resolution.
603 int fps_limit = wants.max_framerate_fps;
604 if (last_frame_pixels <= 320 * 240) {
605 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
606 } else if (last_frame_pixels <= 480 * 270) {
607 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
608 } else if (last_frame_pixels <= 640 * 480) {
609 EXPECT_LE(15, fps_limit);
610 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200611 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800612 }
613 }
614
sprang4847ae62017-06-27 07:06:52 -0700615 void WaitForEncodedFrame(int64_t expected_ntp_time) {
616 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200617 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700618 }
619
620 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
621 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200622 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700623 return ok;
624 }
625
626 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
627 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200628 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700629 }
630
631 void ExpectDroppedFrame() {
632 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200633 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700634 }
635
636 bool WaitForFrame(int64_t timeout_ms) {
637 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200638 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700639 return ok;
640 }
641
perkj26091b12016-09-01 01:17:40 -0700642 class TestEncoder : public test::FakeEncoder {
643 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100644 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700645
asaperssonfab67072017-04-04 05:51:49 -0700646 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800647 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700648 return config_;
649 }
650
651 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800652 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700653 block_next_encode_ = true;
654 }
655
Erik Språngaed30702018-11-05 12:57:17 +0100656 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800657 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100658 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100659 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100660 if (quality_scaling_) {
661 info.scaling_settings =
662 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
663 }
664 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100665 for (int i = 0; i < kMaxSpatialLayers; ++i) {
666 if (temporal_layers_supported_[i]) {
667 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
668 info.fps_allocation[i].resize(num_layers);
669 }
670 }
Erik Språngaed30702018-11-05 12:57:17 +0100671 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200672
673 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Erik Språngaed30702018-11-05 12:57:17 +0100674 return info;
kthelgason876222f2016-11-29 01:44:11 -0800675 }
676
Erik Språngb7cb7b52019-02-26 15:52:33 +0100677 int32_t RegisterEncodeCompleteCallback(
678 EncodedImageCallback* callback) override {
679 rtc::CritScope lock(&local_crit_sect_);
680 encoded_image_callback_ = callback;
681 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
682 }
683
perkjfa10b552016-10-02 23:45:26 -0700684 void ContinueEncode() { continue_encode_event_.Set(); }
685
686 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
687 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800688 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700689 EXPECT_EQ(timestamp_, timestamp);
690 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
691 }
692
kthelgason2fc52542017-03-03 00:24:41 -0800693 void SetQualityScaling(bool b) {
694 rtc::CritScope lock(&local_crit_sect_);
695 quality_scaling_ = b;
696 }
kthelgasonad9010c2017-02-14 00:46:51 -0800697
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100698 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
699 rtc::CritScope lock(&local_crit_sect_);
700 is_hardware_accelerated_ = is_hardware_accelerated;
701 }
702
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100703 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
704 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
705 rtc::CritScope lock(&local_crit_sect_);
706 temporal_layers_supported_[spatial_idx] = supported;
707 }
708
Sergey Silkin6456e352019-07-08 17:56:40 +0200709 void SetResolutionBitrateLimits(
710 std::vector<ResolutionBitrateLimits> thresholds) {
711 rtc::CritScope cs(&local_crit_sect_);
712 resolution_bitrate_limits_ = thresholds;
713 }
714
sprangfe627f32017-03-29 08:24:59 -0700715 void ForceInitEncodeFailure(bool force_failure) {
716 rtc::CritScope lock(&local_crit_sect_);
717 force_init_encode_failed_ = force_failure;
718 }
719
Niels Möller6bb5ab92019-01-11 11:11:10 +0100720 void SimulateOvershoot(double rate_factor) {
721 rtc::CritScope lock(&local_crit_sect_);
722 rate_factor_ = rate_factor;
723 }
724
Erik Språngd7329ca2019-02-21 21:19:53 +0100725 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100726 rtc::CritScope lock(&local_crit_sect_);
727 return last_framerate_;
728 }
729
Erik Språngd7329ca2019-02-21 21:19:53 +0100730 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100731 rtc::CritScope lock(&local_crit_sect_);
732 return last_update_rect_;
733 }
734
Niels Möller87e2d782019-03-07 10:18:23 +0100735 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100736 rtc::CritScope lock(&local_crit_sect_);
737 return last_frame_types_;
738 }
739
740 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100741 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100742 keyframe ? VideoFrameType::kVideoFrameKey
743 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100744 {
745 rtc::CritScope lock(&local_crit_sect_);
746 last_frame_types_ = frame_type;
747 }
Niels Möllerb859b322019-03-07 12:40:01 +0100748 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100749 }
750
Erik Språngb7cb7b52019-02-26 15:52:33 +0100751 void InjectEncodedImage(const EncodedImage& image) {
752 rtc::CritScope lock(&local_crit_sect_);
753 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
754 }
755
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200756 void InjectEncodedImage(const EncodedImage& image,
757 const CodecSpecificInfo* codec_specific_info,
758 const RTPFragmentationHeader* fragmentation) {
759 rtc::CritScope lock(&local_crit_sect_);
760 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
761 fragmentation);
762 }
763
Erik Språngd7329ca2019-02-21 21:19:53 +0100764 void ExpectNullFrame() {
765 rtc::CritScope lock(&local_crit_sect_);
766 expect_null_frame_ = true;
767 }
768
769 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
770 auto allocation = last_bitrate_allocation_;
771 last_bitrate_allocation_.reset();
772 return allocation;
773 }
774
Sergey Silkin5ee69672019-07-02 14:18:34 +0200775 int GetNumEncoderInitializations() const {
776 rtc::CritScope lock(&local_crit_sect_);
777 return num_encoder_initializations_;
778 }
779
perkjfa10b552016-10-02 23:45:26 -0700780 private:
perkj26091b12016-09-01 01:17:40 -0700781 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100782 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700783 bool block_encode;
784 {
brandtre78d2662017-01-16 05:57:16 -0800785 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100786 if (expect_null_frame_) {
787 EXPECT_EQ(input_image.timestamp(), 0u);
788 EXPECT_EQ(input_image.width(), 1);
789 last_frame_types_ = *frame_types;
790 expect_null_frame_ = false;
791 } else {
792 EXPECT_GT(input_image.timestamp(), timestamp_);
793 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
794 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
795 }
perkj26091b12016-09-01 01:17:40 -0700796
797 timestamp_ = input_image.timestamp();
798 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700799 last_input_width_ = input_image.width();
800 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700801 block_encode = block_next_encode_;
802 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100803 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100804 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700805 }
Niels Möllerb859b322019-03-07 12:40:01 +0100806 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700807 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700808 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700809 return result;
810 }
811
sprangfe627f32017-03-29 08:24:59 -0700812 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200813 const Settings& settings) override {
814 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200815
sprangfe627f32017-03-29 08:24:59 -0700816 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100817 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200818
819 ++num_encoder_initializations_;
820
Erik Språng82fad3d2018-03-21 09:57:23 +0100821 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700822 // Simulate setting up temporal layers, in order to validate the life
823 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100824 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200825 frame_buffer_controller_ =
826 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700827 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100828 if (force_init_encode_failed_) {
829 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700830 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100831 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100832
Erik Språngb7cb7b52019-02-26 15:52:33 +0100833 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700834 return res;
835 }
836
Erik Språngb7cb7b52019-02-26 15:52:33 +0100837 int32_t Release() override {
838 rtc::CritScope lock(&local_crit_sect_);
839 EXPECT_NE(initialized_, EncoderState::kUninitialized);
840 initialized_ = EncoderState::kUninitialized;
841 return FakeEncoder::Release();
842 }
843
Erik Språng16cb8f52019-04-12 13:59:09 +0200844 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100845 rtc::CritScope lock(&local_crit_sect_);
846 VideoBitrateAllocation adjusted_rate_allocation;
847 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
848 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200849 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100850 adjusted_rate_allocation.SetBitrate(
851 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200852 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100853 rate_factor_));
854 }
855 }
856 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200857 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
858 last_bitrate_allocation_ = parameters.bitrate;
859 RateControlParameters adjusted_paramters = parameters;
860 adjusted_paramters.bitrate = adjusted_rate_allocation;
861 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100862 }
863
brandtre78d2662017-01-16 05:57:16 -0800864 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100865 enum class EncoderState {
866 kUninitialized,
867 kInitializationFailed,
868 kInitialized
869 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
870 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700871 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700872 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700873 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
874 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
875 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
876 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
877 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100878 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100879 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700880 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100881 absl::optional<bool>
882 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
883 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700884 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100885 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
886 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100887 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100888 VideoFrame::UpdateRect last_update_rect_
889 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100890 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100891 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100892 EncodedImageCallback* encoded_image_callback_
893 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 11:20:09 +0200894 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200895 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +0200896 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
897 RTC_GUARDED_BY(local_crit_sect_);
perkj26091b12016-09-01 01:17:40 -0700898 };
899
mflodmancc3d4422017-08-03 08:27:51 -0700900 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700901 public:
902 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100903 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700904
perkj26091b12016-09-01 01:17:40 -0700905 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700906 EXPECT_TRUE(
907 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
908 }
909
910 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
911 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700912 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700913 if (!encoded_frame_event_.Wait(timeout_ms))
914 return false;
perkj26091b12016-09-01 01:17:40 -0700915 {
916 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800917 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700918 }
919 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700920 return true;
perkj26091b12016-09-01 01:17:40 -0700921 }
922
sprangb1ca0732017-02-01 08:38:12 -0800923 void WaitForEncodedFrame(uint32_t expected_width,
924 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700925 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100926 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700927 }
928
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100929 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700930 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800931 uint32_t width = 0;
932 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800933 {
934 rtc::CritScope lock(&crit_);
935 width = last_width_;
936 height = last_height_;
937 }
938 EXPECT_EQ(expected_height, height);
939 EXPECT_EQ(expected_width, width);
940 }
941
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200942 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
943 VideoRotation rotation;
944 {
945 rtc::CritScope lock(&crit_);
946 rotation = last_rotation_;
947 }
948 EXPECT_EQ(expected_rotation, rotation);
949 }
950
kthelgason2fc52542017-03-03 00:24:41 -0800951 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800952
sprangc5d62e22017-04-02 23:53:04 -0700953 bool WaitForFrame(int64_t timeout_ms) {
954 return encoded_frame_event_.Wait(timeout_ms);
955 }
956
perkj26091b12016-09-01 01:17:40 -0700957 void SetExpectNoFrames() {
958 rtc::CritScope lock(&crit_);
959 expect_frames_ = false;
960 }
961
asaperssonfab67072017-04-04 05:51:49 -0700962 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200963 rtc::CritScope lock(&crit_);
964 return number_of_reconfigurations_;
965 }
966
asaperssonfab67072017-04-04 05:51:49 -0700967 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200968 rtc::CritScope lock(&crit_);
969 return min_transmit_bitrate_bps_;
970 }
971
Erik Språngd7329ca2019-02-21 21:19:53 +0100972 void SetNumExpectedLayers(size_t num_layers) {
973 rtc::CritScope lock(&crit_);
974 num_expected_layers_ = num_layers;
975 }
976
Erik Språngb7cb7b52019-02-26 15:52:33 +0100977 int64_t GetLastCaptureTimeMs() const {
978 rtc::CritScope lock(&crit_);
979 return last_capture_time_ms_;
980 }
981
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200982 std::vector<uint8_t> GetLastEncodedImageData() {
983 rtc::CritScope lock(&crit_);
984 return std::move(last_encoded_image_data_);
985 }
986
987 RTPFragmentationHeader GetLastFragmentation() {
988 rtc::CritScope lock(&crit_);
989 return std::move(last_fragmentation_);
990 }
991
perkj26091b12016-09-01 01:17:40 -0700992 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700993 Result OnEncodedImage(
994 const EncodedImage& encoded_image,
995 const CodecSpecificInfo* codec_specific_info,
996 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200997 rtc::CritScope lock(&crit_);
998 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200999 last_encoded_image_data_ = std::vector<uint8_t>(
1000 encoded_image.data(), encoded_image.data() + encoded_image.size());
1001 if (fragmentation) {
1002 last_fragmentation_.CopyFrom(*fragmentation);
1003 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001004 uint32_t timestamp = encoded_image.Timestamp();
1005 if (last_timestamp_ != timestamp) {
1006 num_received_layers_ = 1;
1007 } else {
1008 ++num_received_layers_;
1009 }
1010 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001011 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001012 last_width_ = encoded_image._encodedWidth;
1013 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001014 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001015 if (num_received_layers_ == num_expected_layers_) {
1016 encoded_frame_event_.Set();
1017 }
sprangb1ca0732017-02-01 08:38:12 -08001018 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001019 }
1020
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001021 void OnEncoderConfigurationChanged(
1022 std::vector<VideoStream> streams,
1023 VideoEncoderConfig::ContentType content_type,
1024 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001025 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001026 ++number_of_reconfigurations_;
1027 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1028 }
1029
perkj26091b12016-09-01 01:17:40 -07001030 rtc::CriticalSection crit_;
1031 TestEncoder* test_encoder_;
1032 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001033 std::vector<uint8_t> last_encoded_image_data_;
1034 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001035 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001036 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001037 uint32_t last_height_ = 0;
1038 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001039 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001040 size_t num_expected_layers_ = 1;
1041 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001042 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001043 int number_of_reconfigurations_ = 0;
1044 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001045 };
1046
Sergey Silkin5ee69672019-07-02 14:18:34 +02001047 class VideoBitrateAllocatorProxyFactory
1048 : public VideoBitrateAllocatorFactory {
1049 public:
1050 VideoBitrateAllocatorProxyFactory()
1051 : bitrate_allocator_factory_(
1052 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1053
1054 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1055 const VideoCodec& codec) override {
1056 rtc::CritScope lock(&crit_);
1057 codec_config_ = codec;
1058 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1059 }
1060
1061 VideoCodec codec_config() const {
1062 rtc::CritScope lock(&crit_);
1063 return codec_config_;
1064 }
1065
1066 private:
1067 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1068
1069 rtc::CriticalSection crit_;
1070 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1071 };
1072
perkj26091b12016-09-01 01:17:40 -07001073 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001074 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001075 int codec_width_;
1076 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001077 int max_framerate_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001078 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001079 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001080 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001081 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001082 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001083 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001084 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001085 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -07001086 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -07001087};
1088
mflodmancc3d4422017-08-03 08:27:51 -07001089TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001090 video_stream_encoder_->OnBitrateUpdated(
1091 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001092 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001093 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001094 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001095 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001096 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001097}
1098
mflodmancc3d4422017-08-03 08:27:51 -07001099TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001100 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001101 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001102 // The encoder will cache up to one frame for a short duration. Adding two
1103 // frames means that the first frame will be dropped and the second frame will
1104 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001105 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001106 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001107 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001108
Erik Språng4c6ca302019-04-08 15:14:01 +02001109 video_stream_encoder_->OnBitrateUpdated(
1110 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001111
Sebastian Janssona3177052018-04-10 13:05:49 +02001112 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001113 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001114 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1115
1116 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001117 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001118}
1119
mflodmancc3d4422017-08-03 08:27:51 -07001120TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001121 video_stream_encoder_->OnBitrateUpdated(
1122 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001123 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001124 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001125
Erik Språng4c6ca302019-04-08 15:14:01 +02001126 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01001127 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001128 // The encoder will cache up to one frame for a short duration. Adding two
1129 // frames means that the first frame will be dropped and the second frame will
1130 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001131 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001132 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001133
Erik Språng4c6ca302019-04-08 15:14:01 +02001134 video_stream_encoder_->OnBitrateUpdated(
1135 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001136 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001137 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1138 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001139 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001140}
1141
mflodmancc3d4422017-08-03 08:27:51 -07001142TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001143 video_stream_encoder_->OnBitrateUpdated(
1144 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001145 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001146 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001147
1148 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001149 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001150
perkja49cbd32016-09-16 07:53:41 -07001151 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001152 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001153 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001154}
1155
mflodmancc3d4422017-08-03 08:27:51 -07001156TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001157 video_stream_encoder_->OnBitrateUpdated(
1158 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001159
perkja49cbd32016-09-16 07:53:41 -07001160 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001161 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001162
mflodmancc3d4422017-08-03 08:27:51 -07001163 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001164 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001165 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001166 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1167 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001168}
1169
mflodmancc3d4422017-08-03 08:27:51 -07001170TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001171 video_stream_encoder_->OnBitrateUpdated(
1172 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001173
1174 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001175 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001176 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001177 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1178 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001179 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1180 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001181 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001182 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001183
mflodmancc3d4422017-08-03 08:27:51 -07001184 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001185}
1186
Noah Richards51db4212019-06-12 06:59:12 -07001187TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1188 video_stream_encoder_->OnBitrateUpdated(
1189 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1190
1191 rtc::Event frame_destroyed_event;
1192 video_source_.IncomingCapturedFrame(
1193 CreateFakeNativeFrame(1, &frame_destroyed_event));
1194 ExpectDroppedFrame();
1195 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1196 video_stream_encoder_->Stop();
1197}
1198
1199TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1200 // Use the cropping factory.
1201 video_encoder_config_.video_stream_factory =
1202 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1203 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1204 kMaxPayloadLength);
1205 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1206
1207 // Capture a frame at codec_width_/codec_height_.
1208 video_stream_encoder_->OnBitrateUpdated(
1209 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1210 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1211 WaitForEncodedFrame(1);
1212 // The encoder will have been configured once.
1213 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1214 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1215 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1216
1217 // Now send in a fake frame that needs to be cropped as the width/height
1218 // aren't divisible by 4 (see CreateEncoderStreams above).
1219 rtc::Event frame_destroyed_event;
1220 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1221 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1222 ExpectDroppedFrame();
1223 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1224 video_stream_encoder_->Stop();
1225}
1226
mflodmancc3d4422017-08-03 08:27:51 -07001227TEST_F(VideoStreamEncoderTest,
1228 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001229 video_stream_encoder_->OnBitrateUpdated(
1230 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001231 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001232
1233 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001234 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001235 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001236 // The encoder will have been configured once when the first frame is
1237 // received.
1238 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001239
1240 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001241 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001242 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001243 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001244 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001245
1246 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001247 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001248 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001249 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001250 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001251
mflodmancc3d4422017-08-03 08:27:51 -07001252 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001253}
1254
mflodmancc3d4422017-08-03 08:27:51 -07001255TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001256 video_stream_encoder_->OnBitrateUpdated(
1257 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001258
1259 // Capture a frame and wait for it to synchronize with the encoder thread.
1260 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001261 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001262 // The encoder will have been configured once.
1263 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001264 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1265 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1266
1267 codec_width_ *= 2;
1268 codec_height_ *= 2;
1269 // Capture a frame with a higher resolution and wait for it to synchronize
1270 // with the encoder thread.
1271 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001272 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001273 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1274 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001275 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001276
mflodmancc3d4422017-08-03 08:27:51 -07001277 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001278}
1279
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001280TEST_F(VideoStreamEncoderTest,
1281 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1282 video_stream_encoder_->OnBitrateUpdated(
1283 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1284
1285 // Capture a frame and wait for it to synchronize with the encoder thread.
1286 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1287 WaitForEncodedFrame(1);
1288
1289 VideoEncoderConfig video_encoder_config;
1290 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1291 // Changing the max payload data length recreates encoder.
1292 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1293 kMaxPayloadLength / 2);
1294
1295 // Capture a frame and wait for it to synchronize with the encoder thread.
1296 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1297 WaitForEncodedFrame(2);
1298 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1299
1300 video_stream_encoder_->Stop();
1301}
1302
Sergey Silkin5ee69672019-07-02 14:18:34 +02001303TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1304 video_stream_encoder_->OnBitrateUpdated(
1305 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1306
1307 VideoEncoderConfig video_encoder_config;
1308 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1309 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1310 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1311 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1312 kMaxPayloadLength);
1313
1314 // Capture a frame and wait for it to synchronize with the encoder thread.
1315 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1316 WaitForEncodedFrame(1);
1317 // The encoder will have been configured once when the first frame is
1318 // received.
1319 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1320 EXPECT_EQ(kTargetBitrateBps,
1321 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1322 EXPECT_EQ(kStartBitrateBps,
1323 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1324
Sergey Silkin6456e352019-07-08 17:56:40 +02001325 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1326 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001327 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1328 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1329 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1330 kMaxPayloadLength);
1331
1332 // Capture a frame and wait for it to synchronize with the encoder thread.
1333 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1334 WaitForEncodedFrame(2);
1335 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1336 // Bitrate limits have changed - rate allocator should be reconfigured,
1337 // encoder should not be reconfigured.
1338 EXPECT_EQ(kTargetBitrateBps * 2,
1339 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1340 EXPECT_EQ(kStartBitrateBps * 2,
1341 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1342 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1343
1344 video_stream_encoder_->Stop();
1345}
1346
Sergey Silkin6456e352019-07-08 17:56:40 +02001347TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001348 EncoderRecommendedBitrateLimitsDoNotOverrideAppBitrateLimits) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001349 video_stream_encoder_->OnBitrateUpdated(
1350 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1351
Sergey Silkin6456e352019-07-08 17:56:40 +02001352 VideoEncoderConfig video_encoder_config;
1353 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1354 video_encoder_config.max_bitrate_bps = 0;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001355 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001356 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1357 kMaxPayloadLength);
1358
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001359 video_source_.IncomingCapturedFrame(CreateFrame(1, 360, 180));
Sergey Silkin6456e352019-07-08 17:56:40 +02001360 WaitForEncodedFrame(1);
Sergey Silkin6456e352019-07-08 17:56:40 +02001361
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001362 // Get the default bitrate limits and use them as baseline for custom
1363 // application and encoder recommended limits.
1364 const uint32_t kDefaultMinBitrateKbps =
1365 bitrate_allocator_factory_.codec_config().minBitrate;
1366 const uint32_t kDefaultMaxBitrateKbps =
1367 bitrate_allocator_factory_.codec_config().maxBitrate;
1368 const uint32_t kEncMinBitrateKbps = kDefaultMinBitrateKbps * 2;
1369 const uint32_t kEncMaxBitrateKbps = kDefaultMaxBitrateKbps * 2;
1370 const uint32_t kAppMinBitrateKbps = kDefaultMinBitrateKbps * 3;
1371 const uint32_t kAppMaxBitrateKbps = kDefaultMaxBitrateKbps * 3;
1372
1373 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1374 codec_width_ * codec_height_, kEncMinBitrateKbps * 1000,
1375 kEncMinBitrateKbps * 1000, kEncMaxBitrateKbps * 1000);
1376 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1377
1378 // Change resolution. This will trigger encoder re-configuration and video
1379 // stream encoder will pick up the bitrate limits recommended by encoder.
1380 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1381 WaitForEncodedFrame(2);
1382 video_source_.IncomingCapturedFrame(CreateFrame(3, 360, 180));
1383 WaitForEncodedFrame(3);
1384
1385 // App bitrate limits are not set - bitrate limits recommended by encoder
1386 // should be used.
1387 EXPECT_EQ(kEncMaxBitrateKbps,
1388 bitrate_allocator_factory_.codec_config().maxBitrate);
1389 EXPECT_EQ(kEncMinBitrateKbps,
1390 bitrate_allocator_factory_.codec_config().minBitrate);
1391
1392 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1393 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
1394 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1395 kMaxPayloadLength);
1396 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1397 WaitForEncodedFrame(4);
1398
1399 // App limited the max bitrate - bitrate limits recommended by encoder should
1400 // not be applied.
1401 EXPECT_EQ(kAppMaxBitrateKbps,
1402 bitrate_allocator_factory_.codec_config().maxBitrate);
1403 EXPECT_EQ(kDefaultMinBitrateKbps,
1404 bitrate_allocator_factory_.codec_config().minBitrate);
1405
1406 video_encoder_config.max_bitrate_bps = 0;
1407 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1408 kAppMinBitrateKbps * 1000;
1409 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1410 kMaxPayloadLength);
1411 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1412 WaitForEncodedFrame(5);
1413
1414 // App limited the min bitrate - bitrate limits recommended by encoder should
1415 // not be applied.
1416 EXPECT_EQ(kDefaultMaxBitrateKbps,
1417 bitrate_allocator_factory_.codec_config().maxBitrate);
1418 EXPECT_EQ(kAppMinBitrateKbps,
1419 bitrate_allocator_factory_.codec_config().minBitrate);
1420
1421 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1422 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1423 kAppMinBitrateKbps * 1000;
Sergey Silkin6456e352019-07-08 17:56:40 +02001424 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1425 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001426 video_source_.IncomingCapturedFrame(CreateFrame(6, nullptr));
1427 WaitForEncodedFrame(6);
Sergey Silkin6456e352019-07-08 17:56:40 +02001428
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001429 // App limited both min and max bitrates - bitrate limits recommended by
1430 // encoder should not be applied.
1431 EXPECT_EQ(kAppMaxBitrateKbps,
1432 bitrate_allocator_factory_.codec_config().maxBitrate);
1433 EXPECT_EQ(kAppMinBitrateKbps,
1434 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001435
1436 video_stream_encoder_->Stop();
1437}
1438
1439TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001440 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001441 video_stream_encoder_->OnBitrateUpdated(
1442 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1443
1444 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001445 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001446 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001447 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001448 fake_encoder_.SetResolutionBitrateLimits(
1449 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1450
1451 VideoEncoderConfig video_encoder_config;
1452 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1453 video_encoder_config.max_bitrate_bps = 0;
1454 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1455 kMaxPayloadLength);
1456
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001457 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001458 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1459 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001460 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1461 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001462 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1463 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1464
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001465 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001466 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1467 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001468 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1469 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001470 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1471 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1472
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001473 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001474 // encoder for 360p should be used.
1475 video_source_.IncomingCapturedFrame(
1476 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1477 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001478 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1479 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001480 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1481 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1482
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001483 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001484 // ignored.
1485 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1486 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001487 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1488 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001489 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1490 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001491 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1492 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001493 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1494 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1495
1496 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1497 // for 270p should be used.
1498 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1499 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001500 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1501 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001502 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1503 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1504
1505 video_stream_encoder_->Stop();
1506}
1507
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001508TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1509 video_stream_encoder_->OnBitrateUpdated(
1510 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1511
1512 VideoEncoderConfig video_encoder_config;
1513 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1514 video_encoder_config.max_bitrate_bps = 0;
1515 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1516 kMaxPayloadLength);
1517
1518 // Encode 720p frame to get the default encoder target bitrate.
1519 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1520 WaitForEncodedFrame(1);
1521 const uint32_t kDefaultTargetBitrateFor720pKbps =
1522 bitrate_allocator_factory_.codec_config()
1523 .simulcastStream[0]
1524 .targetBitrate;
1525
1526 // Set the max recommended encoder bitrate to something lower than the default
1527 // target bitrate.
1528 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1529 1280 * 720, 10 * 1000, 10 * 1000,
1530 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1531 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1532
1533 // Change resolution to trigger encoder reinitialization.
1534 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1535 WaitForEncodedFrame(2);
1536 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1537 WaitForEncodedFrame(3);
1538
1539 // Ensure the target bitrate is capped by the max bitrate.
1540 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1541 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1542 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1543 .simulcastStream[0]
1544 .targetBitrate *
1545 1000,
1546 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1547
1548 video_stream_encoder_->Stop();
1549}
1550
mflodmancc3d4422017-08-03 08:27:51 -07001551TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001552 EXPECT_TRUE(video_source_.has_sinks());
1553 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001554 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001555 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001556 EXPECT_FALSE(video_source_.has_sinks());
1557 EXPECT_TRUE(new_video_source.has_sinks());
1558
mflodmancc3d4422017-08-03 08:27:51 -07001559 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001560}
1561
mflodmancc3d4422017-08-03 08:27:51 -07001562TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001563 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001564 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001565 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001566 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001567}
1568
Jonathan Yubc771b72017-12-08 17:04:29 -08001569TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1570 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001571 const int kWidth = 1280;
1572 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001573
1574 // We rely on the automatic resolution adaptation, but we handle framerate
1575 // adaptation manually by mocking the stats proxy.
1576 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001577
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001578 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001579 video_stream_encoder_->OnBitrateUpdated(
1580 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001581 video_stream_encoder_->SetSource(&video_source_,
1582 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001583 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001584 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001585 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001586 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1587
Jonathan Yubc771b72017-12-08 17:04:29 -08001588 // Adapt down as far as possible.
1589 rtc::VideoSinkWants last_wants;
1590 int64_t t = 1;
1591 int loop_count = 0;
1592 do {
1593 ++loop_count;
1594 last_wants = video_source_.sink_wants();
1595
1596 // Simulate the framerate we've been asked to adapt to.
1597 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1598 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1599 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1600 mock_stats.input_frame_rate = fps;
1601 stats_proxy_->SetMockStats(mock_stats);
1602
1603 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1604 sink_.WaitForEncodedFrame(t);
1605 t += frame_interval_ms;
1606
mflodmancc3d4422017-08-03 08:27:51 -07001607 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001608 VerifyBalancedModeFpsRange(
1609 video_source_.sink_wants(),
1610 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1611 } while (video_source_.sink_wants().max_pixel_count <
1612 last_wants.max_pixel_count ||
1613 video_source_.sink_wants().max_framerate_fps <
1614 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001615
Jonathan Yubc771b72017-12-08 17:04:29 -08001616 // Verify that we've adapted all the way down.
1617 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001618 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001619 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1620 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001621 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001622 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1623 *video_source_.last_sent_height());
1624 EXPECT_EQ(kMinBalancedFramerateFps,
1625 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001626
Jonathan Yubc771b72017-12-08 17:04:29 -08001627 // Adapt back up the same number of times we adapted down.
1628 for (int i = 0; i < loop_count - 1; ++i) {
1629 last_wants = video_source_.sink_wants();
1630
1631 // Simulate the framerate we've been asked to adapt to.
1632 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1633 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1634 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1635 mock_stats.input_frame_rate = fps;
1636 stats_proxy_->SetMockStats(mock_stats);
1637
1638 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1639 sink_.WaitForEncodedFrame(t);
1640 t += frame_interval_ms;
1641
mflodmancc3d4422017-08-03 08:27:51 -07001642 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001643 VerifyBalancedModeFpsRange(
1644 video_source_.sink_wants(),
1645 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1646 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1647 last_wants.max_pixel_count ||
1648 video_source_.sink_wants().max_framerate_fps >
1649 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001650 }
1651
Åsa Persson8c1bf952018-09-13 10:42:19 +02001652 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001653 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001654 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001655 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1656 EXPECT_EQ((loop_count - 1) * 2,
1657 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001658
mflodmancc3d4422017-08-03 08:27:51 -07001659 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001660}
mflodmancc3d4422017-08-03 08:27:51 -07001661TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001662 video_stream_encoder_->OnBitrateUpdated(
1663 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001664 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001665
sprangc5d62e22017-04-02 23:53:04 -07001666 const int kFrameWidth = 1280;
1667 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001668
Åsa Persson8c1bf952018-09-13 10:42:19 +02001669 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001670
kthelgason5e13d412016-12-01 03:59:51 -08001671 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001672 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001673 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001674 frame_timestamp += kFrameIntervalMs;
1675
perkj803d97f2016-11-01 11:45:46 -07001676 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001677 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001678 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001679 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001680 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001681 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001682
asapersson0944a802017-04-07 00:57:58 -07001683 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001684 // wanted resolution.
1685 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1686 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1687 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001688 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001689
1690 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001691 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001693 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001694
sprangc5d62e22017-04-02 23:53:04 -07001695 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001696 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001697
sprangc5d62e22017-04-02 23:53:04 -07001698 // Force an input frame rate to be available, or the adaptation call won't
1699 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001700 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001701 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001702 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001703 stats_proxy_->SetMockStats(stats);
1704
mflodmancc3d4422017-08-03 08:27:51 -07001705 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001706 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001707 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001708 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001709 frame_timestamp += kFrameIntervalMs;
1710
1711 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001712 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001713 EXPECT_EQ(std::numeric_limits<int>::max(),
1714 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001715 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001716
asapersson02465b82017-04-10 01:12:52 -07001717 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001718 video_stream_encoder_->SetSource(&new_video_source,
1719 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001720 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001721
mflodmancc3d4422017-08-03 08:27:51 -07001722 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001723 new_video_source.IncomingCapturedFrame(
1724 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001725 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001726 frame_timestamp += kFrameIntervalMs;
1727
1728 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001729 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001730
1731 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001732 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001733 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001734 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1735 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001736 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001737 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001738
1739 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001740 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001741 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001742 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1743 EXPECT_EQ(std::numeric_limits<int>::max(),
1744 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001745 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001746
mflodmancc3d4422017-08-03 08:27:51 -07001747 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001748}
1749
mflodmancc3d4422017-08-03 08:27:51 -07001750TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001751 video_stream_encoder_->OnBitrateUpdated(
1752 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001753
asaperssonfab67072017-04-04 05:51:49 -07001754 const int kWidth = 1280;
1755 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001756 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001757 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001758 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1759 EXPECT_FALSE(stats.bw_limited_resolution);
1760 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1761
1762 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001763 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001764 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001765 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001766
1767 stats = stats_proxy_->GetStats();
1768 EXPECT_TRUE(stats.bw_limited_resolution);
1769 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1770
1771 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001772 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001773 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001774 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001775
1776 stats = stats_proxy_->GetStats();
1777 EXPECT_FALSE(stats.bw_limited_resolution);
1778 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1779 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1780
mflodmancc3d4422017-08-03 08:27:51 -07001781 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001782}
1783
mflodmancc3d4422017-08-03 08:27:51 -07001784TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001785 video_stream_encoder_->OnBitrateUpdated(
1786 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001787
1788 const int kWidth = 1280;
1789 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001790 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001791 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001792 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1793 EXPECT_FALSE(stats.cpu_limited_resolution);
1794 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1795
1796 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001797 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001798 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001799 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001800
1801 stats = stats_proxy_->GetStats();
1802 EXPECT_TRUE(stats.cpu_limited_resolution);
1803 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1804
1805 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001806 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001807 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001808 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001809
1810 stats = stats_proxy_->GetStats();
1811 EXPECT_FALSE(stats.cpu_limited_resolution);
1812 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001813 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001814
mflodmancc3d4422017-08-03 08:27:51 -07001815 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001816}
1817
mflodmancc3d4422017-08-03 08:27:51 -07001818TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001819 video_stream_encoder_->OnBitrateUpdated(
1820 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001821
asaperssonfab67072017-04-04 05:51:49 -07001822 const int kWidth = 1280;
1823 const int kHeight = 720;
1824 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001825 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001826 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001827 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001828 EXPECT_FALSE(stats.cpu_limited_resolution);
1829 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1830
asaperssonfab67072017-04-04 05:51:49 -07001831 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001832 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001833 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001834 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001835 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001836 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001837 EXPECT_TRUE(stats.cpu_limited_resolution);
1838 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1839
1840 // Set new source with adaptation still enabled.
1841 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001843 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001844
asaperssonfab67072017-04-04 05:51:49 -07001845 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001846 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001847 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001848 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001849 EXPECT_TRUE(stats.cpu_limited_resolution);
1850 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1851
1852 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001853 video_stream_encoder_->SetSource(&new_video_source,
1854 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001855
asaperssonfab67072017-04-04 05:51:49 -07001856 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001857 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001858 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001859 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001860 EXPECT_FALSE(stats.cpu_limited_resolution);
1861 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1862
1863 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001864 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001865 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001866
asaperssonfab67072017-04-04 05:51:49 -07001867 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001868 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001869 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001870 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001871 EXPECT_TRUE(stats.cpu_limited_resolution);
1872 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1873
asaperssonfab67072017-04-04 05:51:49 -07001874 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001875 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001876 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001877 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001878 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001879 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001880 EXPECT_FALSE(stats.cpu_limited_resolution);
1881 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001882 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001883
mflodmancc3d4422017-08-03 08:27:51 -07001884 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001885}
1886
mflodmancc3d4422017-08-03 08:27:51 -07001887TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001888 video_stream_encoder_->OnBitrateUpdated(
1889 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001890
asaperssonfab67072017-04-04 05:51:49 -07001891 const int kWidth = 1280;
1892 const int kHeight = 720;
1893 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001894 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001895 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001896 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001897 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001898 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001899
1900 // Set new source with adaptation still enabled.
1901 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001902 video_stream_encoder_->SetSource(&new_video_source,
1903 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001904
asaperssonfab67072017-04-04 05:51:49 -07001905 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001906 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001907 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001908 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001909 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001910 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001911
asaperssonfab67072017-04-04 05:51:49 -07001912 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001913 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001914 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001915 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001916 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001917 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001918 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001919 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001920
asaperssonfab67072017-04-04 05:51:49 -07001921 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001922 video_stream_encoder_->SetSource(&new_video_source,
1923 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001924
asaperssonfab67072017-04-04 05:51:49 -07001925 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001926 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001927 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001928 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001929 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001930 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001931
asapersson02465b82017-04-10 01:12:52 -07001932 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001933 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001934 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001935
asaperssonfab67072017-04-04 05:51:49 -07001936 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001937 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001938 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001939 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001940 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001941 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1942 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001943
mflodmancc3d4422017-08-03 08:27:51 -07001944 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001945}
1946
mflodmancc3d4422017-08-03 08:27:51 -07001947TEST_F(VideoStreamEncoderTest,
1948 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001949 video_stream_encoder_->OnBitrateUpdated(
1950 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001951
1952 const int kWidth = 1280;
1953 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001954 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001955 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001956 video_source_.IncomingCapturedFrame(
1957 CreateFrame(timestamp_ms, kWidth, kHeight));
1958 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001959 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1960 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1961 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1962
1963 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001964 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001965 timestamp_ms += kFrameIntervalMs;
1966 video_source_.IncomingCapturedFrame(
1967 CreateFrame(timestamp_ms, kWidth, kHeight));
1968 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001969 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1970 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1971 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1972
1973 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001974 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001975 timestamp_ms += kFrameIntervalMs;
1976 video_source_.IncomingCapturedFrame(
1977 CreateFrame(timestamp_ms, kWidth, kHeight));
1978 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001979 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1980 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1981 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1982
Niels Möller4db138e2018-04-19 09:04:13 +02001983 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001984 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001985
1986 VideoEncoderConfig video_encoder_config;
1987 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1988 // Make format different, to force recreation of encoder.
1989 video_encoder_config.video_format.parameters["foo"] = "foo";
1990 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001991 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001992 timestamp_ms += kFrameIntervalMs;
1993 video_source_.IncomingCapturedFrame(
1994 CreateFrame(timestamp_ms, kWidth, kHeight));
1995 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001996 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1997 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1998 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1999
mflodmancc3d4422017-08-03 08:27:51 -07002000 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002001}
2002
mflodmancc3d4422017-08-03 08:27:51 -07002003TEST_F(VideoStreamEncoderTest,
2004 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002005 video_stream_encoder_->OnBitrateUpdated(
2006 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002007
asapersson0944a802017-04-07 00:57:58 -07002008 const int kWidth = 1280;
2009 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002010 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002011
asaperssonfab67072017-04-04 05:51:49 -07002012 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002013 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002014 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002015 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002016 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002017 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2018
asapersson02465b82017-04-10 01:12:52 -07002019 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002020 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002021 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002022 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002023 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002024 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002025 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002026 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2027
2028 // Set new source with adaptation still enabled.
2029 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002030 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002031 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002032
2033 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002034 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002035 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002036 stats = stats_proxy_->GetStats();
2037 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002038 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002039 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2040
sprangc5d62e22017-04-02 23:53:04 -07002041 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002042 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002043 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002044 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002045 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002046 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002047 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002048 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002049 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002050 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002051 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2052
sprangc5d62e22017-04-02 23:53:04 -07002053 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002054 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002055 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2056 mock_stats.input_frame_rate = 30;
2057 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002058 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002059 stats_proxy_->ResetMockStats();
2060
2061 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002062 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002063 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002064
2065 // Framerate now adapted.
2066 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002067 EXPECT_FALSE(stats.cpu_limited_resolution);
2068 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002069 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2070
2071 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002072 video_stream_encoder_->SetSource(&new_video_source,
2073 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002074 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002075 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002076 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002077
2078 stats = stats_proxy_->GetStats();
2079 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002080 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002081 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2082
2083 // Try to trigger overuse. Should not succeed.
2084 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002085 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002086 stats_proxy_->ResetMockStats();
2087
2088 stats = stats_proxy_->GetStats();
2089 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002090 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002091 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2092
2093 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002094 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002095 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002096 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002097 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002098 stats = stats_proxy_->GetStats();
2099 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002100 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002101 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002102
2103 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002104 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002105 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002106 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002107 stats = stats_proxy_->GetStats();
2108 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002109 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002110 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2111
2112 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002113 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002114 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002115 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002116 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002117 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002118 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002119 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002120 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002121 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002122 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2123
2124 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002125 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002126 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002127 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002128 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002129 stats = stats_proxy_->GetStats();
2130 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002131 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002132 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002133 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002134
mflodmancc3d4422017-08-03 08:27:51 -07002135 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002136}
2137
mflodmancc3d4422017-08-03 08:27:51 -07002138TEST_F(VideoStreamEncoderTest,
2139 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002140 const int kWidth = 1280;
2141 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002142 video_stream_encoder_->OnBitrateUpdated(
2143 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002144
asaperssonfab67072017-04-04 05:51:49 -07002145 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002146 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002147
asaperssonfab67072017-04-04 05:51:49 -07002148 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002149 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002150
asaperssonfab67072017-04-04 05:51:49 -07002151 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002152 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002153
asaperssonfab67072017-04-04 05:51:49 -07002154 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002155 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002156
kthelgason876222f2016-11-29 01:44:11 -08002157 // Expect a scale down.
2158 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002159 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002160
asapersson02465b82017-04-10 01:12:52 -07002161 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002162 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002163 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002164 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002165
asaperssonfab67072017-04-04 05:51:49 -07002166 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002167 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002168 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002169 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002170
asaperssonfab67072017-04-04 05:51:49 -07002171 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002172 EXPECT_EQ(std::numeric_limits<int>::max(),
2173 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002174
asaperssonfab67072017-04-04 05:51:49 -07002175 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002176 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002177 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002178 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002179
asapersson02465b82017-04-10 01:12:52 -07002180 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002181 EXPECT_EQ(std::numeric_limits<int>::max(),
2182 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002183
mflodmancc3d4422017-08-03 08:27:51 -07002184 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002185}
2186
mflodmancc3d4422017-08-03 08:27:51 -07002187TEST_F(VideoStreamEncoderTest,
2188 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002189 const int kWidth = 1280;
2190 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002191 video_stream_encoder_->OnBitrateUpdated(
2192 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002193
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002194 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002195 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002196 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002197 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002198
2199 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002200 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002201 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002202 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2203 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2204
2205 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002206 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002207 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002208 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2209 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2210 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2211
2212 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002213 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002214 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2215 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2216 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2217
mflodmancc3d4422017-08-03 08:27:51 -07002218 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002219}
2220
mflodmancc3d4422017-08-03 08:27:51 -07002221TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002222 const int kWidth = 1280;
2223 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002224 video_stream_encoder_->OnBitrateUpdated(
2225 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002226
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002227 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002228 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002229 video_stream_encoder_->SetSource(&source,
2230 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002231 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2232 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002233 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002234
2235 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002236 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002237 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2238 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2239 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2240 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2241
2242 // Trigger adapt down for same input resolution, expect no change.
2243 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2244 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002245 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002246 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2247 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2248 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2249
2250 // Trigger adapt down for larger input resolution, expect no change.
2251 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2252 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002253 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002254 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2255 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2256 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2257
mflodmancc3d4422017-08-03 08:27:51 -07002258 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002259}
2260
mflodmancc3d4422017-08-03 08:27:51 -07002261TEST_F(VideoStreamEncoderTest,
2262 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002263 const int kWidth = 1280;
2264 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002265 video_stream_encoder_->OnBitrateUpdated(
2266 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002267
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002268 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002269 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002270 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002271 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002272
2273 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002274 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002275 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002276 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2277 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2278
2279 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002280 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002281 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002282 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2283 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2284
mflodmancc3d4422017-08-03 08:27:51 -07002285 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002286}
2287
mflodmancc3d4422017-08-03 08:27:51 -07002288TEST_F(VideoStreamEncoderTest,
2289 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002290 const int kWidth = 1280;
2291 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002292 video_stream_encoder_->OnBitrateUpdated(
2293 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002294
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002295 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002296 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002297 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002298 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002299
2300 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002301 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002302 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002303 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002304 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2305
2306 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002307 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002308 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002309 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002310 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2311
mflodmancc3d4422017-08-03 08:27:51 -07002312 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002313}
2314
mflodmancc3d4422017-08-03 08:27:51 -07002315TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002316 const int kWidth = 1280;
2317 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002318 video_stream_encoder_->OnBitrateUpdated(
2319 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002320
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002321 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002322 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002323 video_stream_encoder_->SetSource(&source,
2324 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002325
2326 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2327 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002328 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002329 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2330 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2331 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2332
2333 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002334 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002335 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002336 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2337 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2338 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2339
mflodmancc3d4422017-08-03 08:27:51 -07002340 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002341}
2342
mflodmancc3d4422017-08-03 08:27:51 -07002343TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002344 const int kWidth = 1280;
2345 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002346 video_stream_encoder_->OnBitrateUpdated(
2347 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002348
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002349 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002350 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002351 video_stream_encoder_->SetSource(&source,
2352 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002353
2354 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2355 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002356 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002357 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2358 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2359 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2360
2361 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002362 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002363 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002364 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2365 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2366 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2367
mflodmancc3d4422017-08-03 08:27:51 -07002368 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002369}
2370
mflodmancc3d4422017-08-03 08:27:51 -07002371TEST_F(VideoStreamEncoderTest,
2372 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002373 const int kWidth = 1280;
2374 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002375 video_stream_encoder_->OnBitrateUpdated(
2376 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002377
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002378 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002379 AdaptingFrameForwarder source;
2380 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002381 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002382 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002383
2384 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002385 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002386 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002387 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2388 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2389
2390 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002391 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002392 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002393 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002394 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002395 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2396 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2397
2398 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002399 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002400 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2402 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2403 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2404
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002406}
2407
mflodmancc3d4422017-08-03 08:27:51 -07002408TEST_F(VideoStreamEncoderTest,
2409 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002410 const int kWidth = 1280;
2411 const int kHeight = 720;
2412 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002413 video_stream_encoder_->OnBitrateUpdated(
2414 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002415
2416 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2417 stats.input_frame_rate = kInputFps;
2418 stats_proxy_->SetMockStats(stats);
2419
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002420 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002421 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2422 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002423 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002424
2425 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002426 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002427 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2428 sink_.WaitForEncodedFrame(2);
2429 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2430
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002431 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002432 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002433 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002434 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002435 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002436
2437 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002438 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002439 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2440 sink_.WaitForEncodedFrame(3);
2441 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2442
2443 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002444 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002445 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002446
mflodmancc3d4422017-08-03 08:27:51 -07002447 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002448}
2449
mflodmancc3d4422017-08-03 08:27:51 -07002450TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002451 const int kWidth = 1280;
2452 const int kHeight = 720;
2453 const size_t kNumFrames = 10;
2454
Erik Språng4c6ca302019-04-08 15:14:01 +02002455 video_stream_encoder_->OnBitrateUpdated(
2456 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002457
asaperssond0de2952017-04-21 01:47:31 -07002458 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002459 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002460 video_source_.set_adaptation_enabled(true);
2461
2462 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2463 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2464
2465 int downscales = 0;
2466 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002467 video_source_.IncomingCapturedFrame(
2468 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2469 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002470
asaperssonfab67072017-04-04 05:51:49 -07002471 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002472 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002473 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002474 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002475
2476 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2477 ++downscales;
2478
2479 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2480 EXPECT_EQ(downscales,
2481 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2482 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002483 }
mflodmancc3d4422017-08-03 08:27:51 -07002484 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002485}
2486
mflodmancc3d4422017-08-03 08:27:51 -07002487TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002488 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2489 const int kWidth = 1280;
2490 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002491 video_stream_encoder_->OnBitrateUpdated(
2492 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002493
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002494 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002495 AdaptingFrameForwarder source;
2496 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002497 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002498 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002499
Åsa Persson8c1bf952018-09-13 10:42:19 +02002500 int64_t timestamp_ms = kFrameIntervalMs;
2501 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002502 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002503 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002504 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2505 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2506
2507 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002508 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002509 timestamp_ms += kFrameIntervalMs;
2510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2511 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002512 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002513 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2514 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2515
2516 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002517 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002518 timestamp_ms += kFrameIntervalMs;
2519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002520 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002521 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002522 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2523 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2524
2525 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002526 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002527 timestamp_ms += kFrameIntervalMs;
2528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2529 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002530 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002531 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2532 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2533
2534 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002535 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002536 timestamp_ms += kFrameIntervalMs;
2537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002538 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002539 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002540 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2541 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2542
mflodmancc3d4422017-08-03 08:27:51 -07002543 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002544}
2545
mflodmancc3d4422017-08-03 08:27:51 -07002546TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002547 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2548 const int kWidth = 1280;
2549 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002550 video_stream_encoder_->OnBitrateUpdated(
2551 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002552
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002554 AdaptingFrameForwarder source;
2555 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002556 video_stream_encoder_->SetSource(&source,
2557 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002558
Åsa Persson8c1bf952018-09-13 10:42:19 +02002559 int64_t timestamp_ms = kFrameIntervalMs;
2560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002561 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002562 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2564 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2565
2566 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002567 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002568 timestamp_ms += kFrameIntervalMs;
2569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2570 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002571 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2573 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2574
2575 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002576 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002577 timestamp_ms += kFrameIntervalMs;
2578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002579 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002580 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2582 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2583
2584 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002585 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002586 timestamp_ms += kFrameIntervalMs;
2587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2588 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002589 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2591 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2592
2593 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002594 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002595 timestamp_ms += kFrameIntervalMs;
2596 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002597 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002598 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002599 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2600 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2601
mflodmancc3d4422017-08-03 08:27:51 -07002602 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002603}
2604
Åsa Persson1b247f12019-08-14 17:26:39 +02002605TEST_F(VideoStreamEncoderTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
2606 webrtc::test::ScopedFieldTrials field_trials(
2607 "WebRTC-Video-BalancedDegradationSettings/"
2608 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
2609 // Reset encoder for field trials to take effect.
2610 ConfigureEncoder(video_encoder_config_.Copy());
2611
2612 const int kWidth = 640; // pixels:640x360=230400
2613 const int kHeight = 360;
2614 const int64_t kFrameIntervalMs = 150;
2615 const int kMinBitrateBps = 425000;
2616 const int kTooLowMinBitrateBps = 424000;
2617 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTooLowMinBitrateBps),
2618 DataRate::bps(kTooLowMinBitrateBps),
2619 0, 0);
2620
2621 // Enable BALANCED preference, no initial limitation.
2622 AdaptingFrameForwarder source;
2623 source.set_adaptation_enabled(true);
2624 video_stream_encoder_->SetSource(&source,
2625 webrtc::DegradationPreference::BALANCED);
2626
2627 int64_t timestamp_ms = kFrameIntervalMs;
2628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2629 sink_.WaitForEncodedFrame(kWidth, kHeight);
2630 VerifyFpsMaxResolutionMax(source.sink_wants());
2631 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2632 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2633 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2634
2635 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2636 video_stream_encoder_->TriggerQualityLow();
2637 timestamp_ms += kFrameIntervalMs;
2638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2639 sink_.WaitForEncodedFrame(timestamp_ms);
2640 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2642 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2643 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2644
2645 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2646 video_stream_encoder_->TriggerQualityLow();
2647 timestamp_ms += kFrameIntervalMs;
2648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2649 sink_.WaitForEncodedFrame(timestamp_ms);
2650 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2651 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2652 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2653 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2654
2655 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
2656 video_stream_encoder_->TriggerQualityHigh();
2657 timestamp_ms += kFrameIntervalMs;
2658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2659 sink_.WaitForEncodedFrame(timestamp_ms);
2660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2662 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2663
2664 // Trigger adapt up, expect upscaled resolution (target bitrate == min
2665 // bitrate).
2666 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
2667 DataRate::bps(kMinBitrateBps), 0, 0);
2668 video_stream_encoder_->TriggerQualityHigh();
2669 timestamp_ms += kFrameIntervalMs;
2670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2671 sink_.WaitForEncodedFrame(timestamp_ms);
2672 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2674 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2675
2676 video_stream_encoder_->Stop();
2677}
2678
mflodmancc3d4422017-08-03 08:27:51 -07002679TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002680 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2681 const int kWidth = 1280;
2682 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002683 video_stream_encoder_->OnBitrateUpdated(
2684 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002685
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002686 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002687 AdaptingFrameForwarder source;
2688 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002689 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002690 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002691
Åsa Persson8c1bf952018-09-13 10:42:19 +02002692 int64_t timestamp_ms = kFrameIntervalMs;
2693 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002694 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002695 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002696 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2697 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2698 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2699 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2700
2701 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002702 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002703 timestamp_ms += kFrameIntervalMs;
2704 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2705 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002706 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002707 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2709 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2710 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2711
2712 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002713 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002714 timestamp_ms += kFrameIntervalMs;
2715 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2716 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002717 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002718 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2719 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2720 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2721 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2722
Jonathan Yubc771b72017-12-08 17:04:29 -08002723 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002724 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002725 timestamp_ms += kFrameIntervalMs;
2726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2727 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002728 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002729 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002731 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2733
Jonathan Yubc771b72017-12-08 17:04:29 -08002734 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002735 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002736 timestamp_ms += kFrameIntervalMs;
2737 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2738 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002739 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002740 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002741 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2742 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2743 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2744 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2745
Jonathan Yubc771b72017-12-08 17:04:29 -08002746 // Trigger quality adapt down, expect no change (min resolution reached).
2747 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002748 timestamp_ms += kFrameIntervalMs;
2749 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2750 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002751 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2752 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2753 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2754 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2756
2757 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002758 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002759 timestamp_ms += kFrameIntervalMs;
2760 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2761 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002762 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002763 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2764 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2765 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2766 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2767
2768 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2769 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002770 timestamp_ms += kFrameIntervalMs;
2771 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2772 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002773 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2774 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2775 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2776 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2777 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2778
2779 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2780 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002781 timestamp_ms += kFrameIntervalMs;
2782 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2783 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002784 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002785 last_wants = source.sink_wants();
2786 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2787 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002788 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002789 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2790
2791 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002792 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002793 timestamp_ms += kFrameIntervalMs;
2794 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2795 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002796 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002797 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2798 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002799 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002800 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2801
2802 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002803 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002804 timestamp_ms += kFrameIntervalMs;
2805 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002806 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002807 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002808 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2810 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002811 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002812 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002813
mflodmancc3d4422017-08-03 08:27:51 -07002814 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002815}
2816
mflodmancc3d4422017-08-03 08:27:51 -07002817TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002818 const int kWidth = 640;
2819 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002820
Erik Språng4c6ca302019-04-08 15:14:01 +02002821 video_stream_encoder_->OnBitrateUpdated(
2822 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002823
perkj803d97f2016-11-01 11:45:46 -07002824 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002825 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002826 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002827 }
2828
mflodmancc3d4422017-08-03 08:27:51 -07002829 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002830 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002831 video_source_.IncomingCapturedFrame(CreateFrame(
2832 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002833 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002834 }
2835
mflodmancc3d4422017-08-03 08:27:51 -07002836 video_stream_encoder_->Stop();
2837 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002838 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002839
perkj803d97f2016-11-01 11:45:46 -07002840 EXPECT_EQ(1,
2841 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2842 EXPECT_EQ(
2843 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2844}
2845
mflodmancc3d4422017-08-03 08:27:51 -07002846TEST_F(VideoStreamEncoderTest,
2847 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002848 video_stream_encoder_->OnBitrateUpdated(
2849 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002850 const int kWidth = 640;
2851 const int kHeight = 360;
2852
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002853 video_stream_encoder_->SetSource(&video_source_,
2854 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002855
2856 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2857 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002858 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002859 }
2860
mflodmancc3d4422017-08-03 08:27:51 -07002861 video_stream_encoder_->Stop();
2862 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002863 stats_proxy_.reset();
2864
2865 EXPECT_EQ(0,
2866 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2867}
2868
mflodmancc3d4422017-08-03 08:27:51 -07002869TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002870 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002871 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002872
2873 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002874 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002875 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02002876 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
2877 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08002878
sprang57c2fff2017-01-16 06:24:02 -08002879 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002880 .Times(1);
Erik Språng610c7632019-03-06 15:37:33 +01002881 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02002882 DataRate::bps(kLowTargetBitrateBps),
2883 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002884
sprang57c2fff2017-01-16 06:24:02 -08002885 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002886 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2887 WaitForEncodedFrame(rtc::TimeMillis());
2888 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2889 fake_encoder_.GetAndResetLastBitrateAllocation();
2890 // Check that encoder has been updated too, not just allocation observer.
2891 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02002892 // TODO(srte): The use of millisecs here looks like an error, but the tests
2893 // fails using seconds, this should be investigated.
2894 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002895
2896 // Not called on second frame.
2897 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2898 .Times(0);
2899 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002900 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2901 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002902 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002903
2904 // Called after a process interval.
2905 const int64_t kProcessIntervalMs =
2906 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002907 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2908 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002909 const int64_t start_time_ms = rtc::TimeMillis();
2910 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2911 video_source_.IncomingCapturedFrame(
2912 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2913 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02002914 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01002915 }
2916
2917 // Since rates are unchanged, encoder should not be reconfigured.
2918 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002919
mflodmancc3d4422017-08-03 08:27:51 -07002920 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002921}
2922
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01002923TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
2924 // 2 TLs configured, temporal layers supported by encoder.
2925 const int kNumTemporalLayers = 2;
2926 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
2927 fake_encoder_.SetTemporalLayersSupported(0, true);
2928
2929 // Bitrate allocated across temporal layers.
2930 const int kTl0Bps = kTargetBitrateBps *
2931 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2932 kNumTemporalLayers, /*temporal_id*/ 0);
2933 const int kTl1Bps = kTargetBitrateBps *
2934 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2935 kNumTemporalLayers, /*temporal_id*/ 1);
2936 VideoBitrateAllocation expected_bitrate;
2937 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
2938 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
2939
2940 VerifyAllocatedBitrate(expected_bitrate);
2941 video_stream_encoder_->Stop();
2942}
2943
2944TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
2945 // 2 TLs configured, temporal layers not supported by encoder.
2946 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2947 fake_encoder_.SetTemporalLayersSupported(0, false);
2948
2949 // Temporal layers not supported by the encoder.
2950 // Total bitrate should be at ti:0.
2951 VideoBitrateAllocation expected_bitrate;
2952 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
2953
2954 VerifyAllocatedBitrate(expected_bitrate);
2955 video_stream_encoder_->Stop();
2956}
2957
2958TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
2959 // 2 TLs configured, temporal layers only supported for first stream.
2960 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
2961 fake_encoder_.SetTemporalLayersSupported(0, true);
2962 fake_encoder_.SetTemporalLayersSupported(1, false);
2963
2964 const int kS0Bps = 150000;
2965 const int kS0Tl0Bps =
2966 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2967 /*num_layers*/ 2, /*temporal_id*/ 0);
2968 const int kS0Tl1Bps =
2969 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
2970 /*num_layers*/ 2, /*temporal_id*/ 1);
2971 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
2972 // Temporal layers not supported by si:1.
2973 VideoBitrateAllocation expected_bitrate;
2974 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
2975 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
2976 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
2977
2978 VerifyAllocatedBitrate(expected_bitrate);
2979 video_stream_encoder_->Stop();
2980}
2981
Niels Möller7dc26b72017-12-06 10:27:48 +01002982TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2983 const int kFrameWidth = 1280;
2984 const int kFrameHeight = 720;
2985 const int kFramerate = 24;
2986
Erik Språng4c6ca302019-04-08 15:14:01 +02002987 video_stream_encoder_->OnBitrateUpdated(
2988 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01002989 test::FrameForwarder source;
2990 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002991 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002992
2993 // Insert a single frame, triggering initial configuration.
2994 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2995 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2996
2997 EXPECT_EQ(
2998 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2999 kDefaultFramerate);
3000
3001 // Trigger reconfigure encoder (without resetting the entire instance).
3002 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003003 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003004 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3005 video_encoder_config.number_of_streams = 1;
3006 video_encoder_config.video_stream_factory =
3007 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3008 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003009 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003010 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3011
3012 // Detector should be updated with fps limit from codec config.
3013 EXPECT_EQ(
3014 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3015 kFramerate);
3016
3017 // Trigger overuse, max framerate should be reduced.
3018 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3019 stats.input_frame_rate = kFramerate;
3020 stats_proxy_->SetMockStats(stats);
3021 video_stream_encoder_->TriggerCpuOveruse();
3022 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3023 int adapted_framerate =
3024 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3025 EXPECT_LT(adapted_framerate, kFramerate);
3026
3027 // Trigger underuse, max framerate should go back to codec configured fps.
3028 // Set extra low fps, to make sure it's actually reset, not just incremented.
3029 stats = stats_proxy_->GetStats();
3030 stats.input_frame_rate = adapted_framerate / 2;
3031 stats_proxy_->SetMockStats(stats);
3032 video_stream_encoder_->TriggerCpuNormalUsage();
3033 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3034 EXPECT_EQ(
3035 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3036 kFramerate);
3037
3038 video_stream_encoder_->Stop();
3039}
3040
3041TEST_F(VideoStreamEncoderTest,
3042 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3043 const int kFrameWidth = 1280;
3044 const int kFrameHeight = 720;
3045 const int kLowFramerate = 15;
3046 const int kHighFramerate = 25;
3047
Erik Språng4c6ca302019-04-08 15:14:01 +02003048 video_stream_encoder_->OnBitrateUpdated(
3049 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003050 test::FrameForwarder source;
3051 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003052 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003053
3054 // Trigger initial configuration.
3055 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003056 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003057 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3058 video_encoder_config.number_of_streams = 1;
3059 video_encoder_config.video_stream_factory =
3060 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3061 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3062 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003063 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003064 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3065
3066 EXPECT_EQ(
3067 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3068 kLowFramerate);
3069
3070 // Trigger overuse, max framerate should be reduced.
3071 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3072 stats.input_frame_rate = kLowFramerate;
3073 stats_proxy_->SetMockStats(stats);
3074 video_stream_encoder_->TriggerCpuOveruse();
3075 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3076 int adapted_framerate =
3077 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3078 EXPECT_LT(adapted_framerate, kLowFramerate);
3079
3080 // Reconfigure the encoder with a new (higher max framerate), max fps should
3081 // still respect the adaptation.
3082 video_encoder_config.video_stream_factory =
3083 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3084 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3085 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003086 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003087 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3088
3089 EXPECT_EQ(
3090 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3091 adapted_framerate);
3092
3093 // Trigger underuse, max framerate should go back to codec configured fps.
3094 stats = stats_proxy_->GetStats();
3095 stats.input_frame_rate = adapted_framerate;
3096 stats_proxy_->SetMockStats(stats);
3097 video_stream_encoder_->TriggerCpuNormalUsage();
3098 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3099 EXPECT_EQ(
3100 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3101 kHighFramerate);
3102
3103 video_stream_encoder_->Stop();
3104}
3105
mflodmancc3d4422017-08-03 08:27:51 -07003106TEST_F(VideoStreamEncoderTest,
3107 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003108 const int kFrameWidth = 1280;
3109 const int kFrameHeight = 720;
3110 const int kFramerate = 24;
3111
Erik Språng4c6ca302019-04-08 15:14:01 +02003112 video_stream_encoder_->OnBitrateUpdated(
3113 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003114 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003116 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003117
3118 // Trigger initial configuration.
3119 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003120 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003121 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3122 video_encoder_config.number_of_streams = 1;
3123 video_encoder_config.video_stream_factory =
3124 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3125 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003126 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003127 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003128 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003129
Niels Möller7dc26b72017-12-06 10:27:48 +01003130 EXPECT_EQ(
3131 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3132 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003133
3134 // Trigger overuse, max framerate should be reduced.
3135 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3136 stats.input_frame_rate = kFramerate;
3137 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003138 video_stream_encoder_->TriggerCpuOveruse();
3139 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003140 int adapted_framerate =
3141 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003142 EXPECT_LT(adapted_framerate, kFramerate);
3143
3144 // Change degradation preference to not enable framerate scaling. Target
3145 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003146 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003147 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003148 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003149 EXPECT_EQ(
3150 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3151 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003152
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003154}
3155
mflodmancc3d4422017-08-03 08:27:51 -07003156TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003157 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003158 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003159 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3160 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003161 const int kWidth = 640;
3162 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003163
asaperssonfab67072017-04-04 05:51:49 -07003164 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003165
3166 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003167 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003168
3169 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003170 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003171
sprangc5d62e22017-04-02 23:53:04 -07003172 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003173
asaperssonfab67072017-04-04 05:51:49 -07003174 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003175 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003176 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003177
3178 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003179 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003180
sprangc5d62e22017-04-02 23:53:04 -07003181 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003182
mflodmancc3d4422017-08-03 08:27:51 -07003183 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003184}
3185
mflodmancc3d4422017-08-03 08:27:51 -07003186TEST_F(VideoStreamEncoderTest,
3187 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003188 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003189 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003190 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3191 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003192 const int kWidth = 640;
3193 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003194
3195 // We expect the n initial frames to get dropped.
3196 int i;
3197 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003198 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003199 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003200 }
3201 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003202 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003203 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003204
3205 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003206 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003207
mflodmancc3d4422017-08-03 08:27:51 -07003208 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003209}
3210
mflodmancc3d4422017-08-03 08:27:51 -07003211TEST_F(VideoStreamEncoderTest,
3212 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003213 const int kWidth = 640;
3214 const int kHeight = 360;
Erik Språng610c7632019-03-06 15:37:33 +01003215 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003216 DataRate::bps(kLowTargetBitrateBps),
3217 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003218
3219 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003220 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003221 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003222
asaperssonfab67072017-04-04 05:51:49 -07003223 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003224 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003225 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003226
mflodmancc3d4422017-08-03 08:27:51 -07003227 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003228}
3229
mflodmancc3d4422017-08-03 08:27:51 -07003230TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003231 const int kWidth = 640;
3232 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003233 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003234
3235 VideoEncoderConfig video_encoder_config;
3236 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3237 // Make format different, to force recreation of encoder.
3238 video_encoder_config.video_format.parameters["foo"] = "foo";
3239 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003240 kMaxPayloadLength);
Erik Språng610c7632019-03-06 15:37:33 +01003241 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003242 DataRate::bps(kLowTargetBitrateBps),
3243 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003244
kthelgasonb83797b2017-02-14 11:57:25 -08003245 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003246 video_stream_encoder_->SetSource(&video_source_,
3247 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003248
asaperssonfab67072017-04-04 05:51:49 -07003249 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003250 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003251 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003252
mflodmancc3d4422017-08-03 08:27:51 -07003253 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003254 fake_encoder_.SetQualityScaling(true);
3255}
3256
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003257TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
3258 webrtc::test::ScopedFieldTrials field_trials(
3259 "WebRTC-InitialFramedrop/Enabled/");
3260 // Reset encoder for field trials to take effect.
3261 ConfigureEncoder(video_encoder_config_.Copy());
3262 const int kTooLowBitrateForFrameSizeBps = 10000;
3263 const int kWidth = 640;
3264 const int kHeight = 360;
3265
Erik Språng4c6ca302019-04-08 15:14:01 +02003266 video_stream_encoder_->OnBitrateUpdated(
3267 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003268 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3269 // Frame should not be dropped.
3270 WaitForEncodedFrame(1);
3271
Erik Språng610c7632019-03-06 15:37:33 +01003272 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003273 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3274 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003275 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3276 // Expect to drop this frame, the wait should time out.
3277 ExpectDroppedFrame();
3278
3279 // Expect the sink_wants to specify a scaled frame.
3280 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3281 video_stream_encoder_->Stop();
3282}
3283
Åsa Persson139f4dc2019-08-02 09:29:58 +02003284TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3285 webrtc::test::ScopedFieldTrials field_trials(
3286 "WebRTC-Video-QualityScalerSettings/"
3287 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3288 // Reset encoder for field trials to take effect.
3289 ConfigureEncoder(video_encoder_config_.Copy());
3290 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3291 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3292 const int kWidth = 640;
3293 const int kHeight = 360;
3294
3295 video_stream_encoder_->OnBitrateUpdated(
3296 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
3297 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3298 // Frame should not be dropped.
3299 WaitForEncodedFrame(1);
3300
3301 video_stream_encoder_->OnBitrateUpdated(
3302 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
3303 DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0);
3304 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3305 // Frame should not be dropped.
3306 WaitForEncodedFrame(2);
3307
3308 video_stream_encoder_->OnBitrateUpdated(
3309 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3310 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
3311 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3312 // Expect to drop this frame, the wait should time out.
3313 ExpectDroppedFrame();
3314
3315 // Expect the sink_wants to specify a scaled frame.
3316 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3317 video_stream_encoder_->Stop();
3318}
3319
mflodmancc3d4422017-08-03 08:27:51 -07003320TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003321 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3322 const int kTooSmallWidth = 10;
3323 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02003324 video_stream_encoder_->OnBitrateUpdated(
3325 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003326
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003327 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003328 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003329 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003330 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003331 VerifyNoLimitation(source.sink_wants());
3332 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3333
3334 // Trigger adapt down, too small frame, expect no change.
3335 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003336 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003337 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003338 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3341
mflodmancc3d4422017-08-03 08:27:51 -07003342 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003343}
3344
mflodmancc3d4422017-08-03 08:27:51 -07003345TEST_F(VideoStreamEncoderTest,
3346 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003347 const int kTooSmallWidth = 10;
3348 const int kTooSmallHeight = 10;
3349 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02003350 video_stream_encoder_->OnBitrateUpdated(
3351 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003352
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003353 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003354 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003355 video_stream_encoder_->SetSource(&source,
3356 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003357 VerifyNoLimitation(source.sink_wants());
3358 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3359 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3360
3361 // Trigger adapt down, expect limited framerate.
3362 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003363 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003364 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003365 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3366 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3367 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3368 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3369
3370 // Trigger adapt down, too small frame, expect no change.
3371 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003372 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003373 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003374 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3375 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3376 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3377 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3378
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003380}
3381
mflodmancc3d4422017-08-03 08:27:51 -07003382TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07003383 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02003384 video_stream_encoder_->OnBitrateUpdated(
3385 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02003386 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07003387 const int kFrameWidth = 1280;
3388 const int kFrameHeight = 720;
3389 video_source_.IncomingCapturedFrame(
3390 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003391 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003393}
3394
sprangb1ca0732017-02-01 08:38:12 -08003395// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07003396TEST_F(VideoStreamEncoderTest,
3397 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003398 video_stream_encoder_->OnBitrateUpdated(
3399 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08003400
3401 const int kFrameWidth = 1280;
3402 const int kFrameHeight = 720;
3403 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07003404 // requested by
3405 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08003406 video_source_.set_adaptation_enabled(true);
3407
3408 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003409 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003410 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003411
3412 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003413 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08003414 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003415 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003416 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08003417
asaperssonfab67072017-04-04 05:51:49 -07003418 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003419 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08003420 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003421 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003422 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003423
mflodmancc3d4422017-08-03 08:27:51 -07003424 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08003425}
sprangfe627f32017-03-29 08:24:59 -07003426
mflodmancc3d4422017-08-03 08:27:51 -07003427TEST_F(VideoStreamEncoderTest,
3428 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07003429 const int kFrameWidth = 1280;
3430 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07003431
Erik Språng4c6ca302019-04-08 15:14:01 +02003432 video_stream_encoder_->OnBitrateUpdated(
3433 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003434 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003435 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003436 video_source_.set_adaptation_enabled(true);
3437
sprang4847ae62017-06-27 07:06:52 -07003438 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003439
3440 video_source_.IncomingCapturedFrame(
3441 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003442 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003443
3444 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07003445 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003446
3447 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07003448 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003449 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003450 video_source_.IncomingCapturedFrame(
3451 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003452 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003453 }
3454
3455 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07003456 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003457 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003458 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003459 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003460 video_source_.IncomingCapturedFrame(
3461 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003462 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003463 ++num_frames_dropped;
3464 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003465 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003466 }
3467 }
3468
sprang4847ae62017-06-27 07:06:52 -07003469 // Add some slack to account for frames dropped by the frame dropper.
3470 const int kErrorMargin = 1;
3471 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003472 kErrorMargin);
3473
3474 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07003475 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003476 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003477 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003478 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003479 video_source_.IncomingCapturedFrame(
3480 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003481 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003482 ++num_frames_dropped;
3483 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003484 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003485 }
3486 }
sprang4847ae62017-06-27 07:06:52 -07003487 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07003488 kErrorMargin);
3489
3490 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07003491 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003492 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003493 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003494 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003495 video_source_.IncomingCapturedFrame(
3496 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003497 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003498 ++num_frames_dropped;
3499 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003500 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003501 }
3502 }
sprang4847ae62017-06-27 07:06:52 -07003503 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003504 kErrorMargin);
3505
3506 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07003507 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003508 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003509 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003510 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003511 video_source_.IncomingCapturedFrame(
3512 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003513 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003514 ++num_frames_dropped;
3515 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003516 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003517 }
3518 }
3519 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
3520
mflodmancc3d4422017-08-03 08:27:51 -07003521 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003522}
3523
mflodmancc3d4422017-08-03 08:27:51 -07003524TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07003525 const int kFramerateFps = 5;
3526 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07003527 const int kFrameWidth = 1280;
3528 const int kFrameHeight = 720;
3529
sprang4847ae62017-06-27 07:06:52 -07003530 // Reconfigure encoder with two temporal layers and screensharing, which will
3531 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02003532 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07003533
Erik Språng4c6ca302019-04-08 15:14:01 +02003534 video_stream_encoder_->OnBitrateUpdated(
3535 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003536 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003537 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003538 video_source_.set_adaptation_enabled(true);
3539
sprang4847ae62017-06-27 07:06:52 -07003540 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003541
3542 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08003543 rtc::VideoSinkWants last_wants;
3544 do {
3545 last_wants = video_source_.sink_wants();
3546
sprangc5d62e22017-04-02 23:53:04 -07003547 // Insert frames to get a new fps estimate...
3548 for (int j = 0; j < kFramerateFps; ++j) {
3549 video_source_.IncomingCapturedFrame(
3550 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08003551 if (video_source_.last_sent_width()) {
3552 sink_.WaitForEncodedFrame(timestamp_ms);
3553 }
sprangc5d62e22017-04-02 23:53:04 -07003554 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003555 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07003556 }
3557 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07003558 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003559 } while (video_source_.sink_wants().max_framerate_fps <
3560 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003561
Jonathan Yubc771b72017-12-08 17:04:29 -08003562 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003563
mflodmancc3d4422017-08-03 08:27:51 -07003564 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003565}
asaperssonf7e294d2017-06-13 23:25:22 -07003566
mflodmancc3d4422017-08-03 08:27:51 -07003567TEST_F(VideoStreamEncoderTest,
3568 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003569 const int kWidth = 1280;
3570 const int kHeight = 720;
3571 const int64_t kFrameIntervalMs = 150;
3572 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003573 video_stream_encoder_->OnBitrateUpdated(
3574 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003575
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003576 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003577 AdaptingFrameForwarder source;
3578 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003579 video_stream_encoder_->SetSource(&source,
3580 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003581 timestamp_ms += kFrameIntervalMs;
3582 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003583 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003584 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003585 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3586 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3587 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3588
3589 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003590 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003593 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003594 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3597 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3598
3599 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003600 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003601 timestamp_ms += kFrameIntervalMs;
3602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003603 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003604 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3607 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3608
3609 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003610 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003611 timestamp_ms += kFrameIntervalMs;
3612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003613 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003614 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3615 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3617 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3618
3619 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003620 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003621 timestamp_ms += kFrameIntervalMs;
3622 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003623 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003624 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3625 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3627 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3628
3629 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003630 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003631 timestamp_ms += kFrameIntervalMs;
3632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003633 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003634 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3637 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3638
3639 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003640 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003641 timestamp_ms += kFrameIntervalMs;
3642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003643 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003644 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3647 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3648
3649 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003650 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003651 timestamp_ms += kFrameIntervalMs;
3652 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003653 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003654 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3655 rtc::VideoSinkWants last_wants = source.sink_wants();
3656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3658 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3659
3660 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003661 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003662 timestamp_ms += kFrameIntervalMs;
3663 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003664 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003665 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3666 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3667 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3668 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3669
3670 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003671 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003672 timestamp_ms += kFrameIntervalMs;
3673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003674 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003675 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3676 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3677 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3678 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3679
3680 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003681 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003682 timestamp_ms += kFrameIntervalMs;
3683 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003684 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003685 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3686 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3687 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3688 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3689
3690 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003691 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003692 timestamp_ms += kFrameIntervalMs;
3693 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003694 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003695 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3696 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3697 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3698 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3699
3700 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003701 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003702 timestamp_ms += kFrameIntervalMs;
3703 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003704 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003705 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3706 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3707 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3708 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3709
3710 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003711 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003712 timestamp_ms += kFrameIntervalMs;
3713 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003714 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003715 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3716 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3717 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3718 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3719
3720 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003721 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003722 timestamp_ms += kFrameIntervalMs;
3723 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003724 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003725 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3726 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3727 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3728 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3729
3730 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003731 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003732 timestamp_ms += kFrameIntervalMs;
3733 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003734 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003735 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003736 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3738 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3739 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3740
3741 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003742 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003743 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003744 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3745
mflodmancc3d4422017-08-03 08:27:51 -07003746 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003747}
3748
mflodmancc3d4422017-08-03 08:27:51 -07003749TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003750 const int kWidth = 1280;
3751 const int kHeight = 720;
3752 const int64_t kFrameIntervalMs = 150;
3753 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003754 video_stream_encoder_->OnBitrateUpdated(
3755 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003756
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003757 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003758 AdaptingFrameForwarder source;
3759 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003760 video_stream_encoder_->SetSource(&source,
3761 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003762 timestamp_ms += kFrameIntervalMs;
3763 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003764 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003765 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003766 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3767 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3768 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3769 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3770 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3771 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3772
3773 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003774 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003775 timestamp_ms += kFrameIntervalMs;
3776 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003777 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003778 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3779 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3780 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3781 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3782 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3783 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3784 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3785
3786 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003787 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003788 timestamp_ms += kFrameIntervalMs;
3789 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003790 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003791 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3792 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3793 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3794 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3795 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3796 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3797 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3798
3799 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003800 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003801 timestamp_ms += kFrameIntervalMs;
3802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003803 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003804 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3805 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3806 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3807 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3809 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3810 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3811
3812 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003813 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003814 timestamp_ms += kFrameIntervalMs;
3815 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003816 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003817 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3818 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3819 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3820 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3821 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3822 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3823 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3824
3825 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003826 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003827 timestamp_ms += kFrameIntervalMs;
3828 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003829 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003830 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3831 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3832 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3833 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3834 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3835 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3836 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3837
3838 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003839 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003840 timestamp_ms += kFrameIntervalMs;
3841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003842 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003843 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003844 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003845 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3846 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3847 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3848 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3849 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3850 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3851
3852 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003853 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003854 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003855 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3856 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3857
mflodmancc3d4422017-08-03 08:27:51 -07003858 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003859}
3860
mflodmancc3d4422017-08-03 08:27:51 -07003861TEST_F(VideoStreamEncoderTest,
3862 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003863 const int kWidth = 640;
3864 const int kHeight = 360;
3865 const int kFpsLimit = 15;
3866 const int64_t kFrameIntervalMs = 150;
3867 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003868 video_stream_encoder_->OnBitrateUpdated(
3869 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003870
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003871 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003872 AdaptingFrameForwarder source;
3873 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003874 video_stream_encoder_->SetSource(&source,
3875 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003876 timestamp_ms += kFrameIntervalMs;
3877 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003878 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003879 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003880 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3881 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3883 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3884 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3885 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3886
3887 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003888 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003889 timestamp_ms += kFrameIntervalMs;
3890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003891 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003892 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3893 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3895 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3896 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3897 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3898 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3899
3900 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003901 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003902 timestamp_ms += kFrameIntervalMs;
3903 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003904 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003905 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3907 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3908 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3909 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3910 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3911 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3912
3913 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003914 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003915 timestamp_ms += kFrameIntervalMs;
3916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003917 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003918 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3923 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3924 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3925
3926 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003927 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003928 timestamp_ms += kFrameIntervalMs;
3929 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003930 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003931 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003932 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3933 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3934 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3935 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3936 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3937 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3938
3939 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003940 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003941 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003942 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3943 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3944
mflodmancc3d4422017-08-03 08:27:51 -07003945 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003946}
3947
mflodmancc3d4422017-08-03 08:27:51 -07003948TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003949 const int kFrameWidth = 1920;
3950 const int kFrameHeight = 1080;
3951 // 3/4 of 1920.
3952 const int kAdaptedFrameWidth = 1440;
3953 // 3/4 of 1080 rounded down to multiple of 4.
3954 const int kAdaptedFrameHeight = 808;
3955 const int kFramerate = 24;
3956
Erik Språng4c6ca302019-04-08 15:14:01 +02003957 video_stream_encoder_->OnBitrateUpdated(
3958 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003959 // Trigger reconfigure encoder (without resetting the entire instance).
3960 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003961 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003962 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3963 video_encoder_config.number_of_streams = 1;
3964 video_encoder_config.video_stream_factory =
3965 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003966 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003967 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003968 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003969
3970 video_source_.set_adaptation_enabled(true);
3971
3972 video_source_.IncomingCapturedFrame(
3973 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003974 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003975
3976 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003977 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003978 video_source_.IncomingCapturedFrame(
3979 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003980 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003981
mflodmancc3d4422017-08-03 08:27:51 -07003982 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003983}
3984
mflodmancc3d4422017-08-03 08:27:51 -07003985TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003986 const int kFrameWidth = 1280;
3987 const int kFrameHeight = 720;
3988 const int kLowFps = 2;
3989 const int kHighFps = 30;
3990
Erik Språng4c6ca302019-04-08 15:14:01 +02003991 video_stream_encoder_->OnBitrateUpdated(
3992 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003993
3994 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3995 max_framerate_ = kLowFps;
3996
3997 // Insert 2 seconds of 2fps video.
3998 for (int i = 0; i < kLowFps * 2; ++i) {
3999 video_source_.IncomingCapturedFrame(
4000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4001 WaitForEncodedFrame(timestamp_ms);
4002 timestamp_ms += 1000 / kLowFps;
4003 }
4004
4005 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004006 video_stream_encoder_->OnBitrateUpdated(
4007 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004008 video_source_.IncomingCapturedFrame(
4009 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4010 WaitForEncodedFrame(timestamp_ms);
4011 timestamp_ms += 1000 / kLowFps;
4012
4013 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4014
4015 // Insert 30fps frames for just a little more than the forced update period.
4016 const int kVcmTimerIntervalFrames =
4017 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
4018 const int kFrameIntervalMs = 1000 / kHighFps;
4019 max_framerate_ = kHighFps;
4020 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4021 video_source_.IncomingCapturedFrame(
4022 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4023 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4024 // be dropped if the encoder hans't been updated with the new higher target
4025 // framerate yet, causing it to overshoot the target bitrate and then
4026 // suffering the wrath of the media optimizer.
4027 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4028 timestamp_ms += kFrameIntervalMs;
4029 }
4030
4031 // Don expect correct measurement just yet, but it should be higher than
4032 // before.
4033 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4034
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004036}
4037
mflodmancc3d4422017-08-03 08:27:51 -07004038TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004039 const int kFrameWidth = 1280;
4040 const int kFrameHeight = 720;
4041 const int kTargetBitrateBps = 1000000;
4042
4043 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004044 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004045 video_stream_encoder_->OnBitrateUpdated(
4046 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004047 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004048
4049 // Insert a first video frame, causes another bitrate update.
4050 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4051 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4052 video_source_.IncomingCapturedFrame(
4053 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4054 WaitForEncodedFrame(timestamp_ms);
4055
4056 // Next, simulate video suspension due to pacer queue overrun.
Erik Språng4c6ca302019-04-08 15:14:01 +02004057 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01004058 1);
sprang4847ae62017-06-27 07:06:52 -07004059
4060 // Skip ahead until a new periodic parameter update should have occured.
4061 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02004062 fake_clock_.AdvanceTime(
4063 TimeDelta::ms(vcm::VCMProcessTimer::kDefaultProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004064
4065 // Bitrate observer should not be called.
4066 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4067 video_source_.IncomingCapturedFrame(
4068 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4069 ExpectDroppedFrame();
4070
mflodmancc3d4422017-08-03 08:27:51 -07004071 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004072}
ilnik6b826ef2017-06-16 06:53:48 -07004073
Niels Möller4db138e2018-04-19 09:04:13 +02004074TEST_F(VideoStreamEncoderTest,
4075 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4076 const int kFrameWidth = 1280;
4077 const int kFrameHeight = 720;
4078 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004079 video_stream_encoder_->OnBitrateUpdated(
4080 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004081 video_source_.IncomingCapturedFrame(
4082 CreateFrame(1, kFrameWidth, kFrameHeight));
4083 WaitForEncodedFrame(1);
4084 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4085 .low_encode_usage_threshold_percent,
4086 default_options.low_encode_usage_threshold_percent);
4087 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4088 .high_encode_usage_threshold_percent,
4089 default_options.high_encode_usage_threshold_percent);
4090 video_stream_encoder_->Stop();
4091}
4092
4093TEST_F(VideoStreamEncoderTest,
4094 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4095 const int kFrameWidth = 1280;
4096 const int kFrameHeight = 720;
4097 CpuOveruseOptions hardware_options;
4098 hardware_options.low_encode_usage_threshold_percent = 150;
4099 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004100 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004101
Erik Språng4c6ca302019-04-08 15:14:01 +02004102 video_stream_encoder_->OnBitrateUpdated(
4103 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004104 video_source_.IncomingCapturedFrame(
4105 CreateFrame(1, kFrameWidth, kFrameHeight));
4106 WaitForEncodedFrame(1);
4107 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4108 .low_encode_usage_threshold_percent,
4109 hardware_options.low_encode_usage_threshold_percent);
4110 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4111 .high_encode_usage_threshold_percent,
4112 hardware_options.high_encode_usage_threshold_percent);
4113 video_stream_encoder_->Stop();
4114}
4115
Niels Möller6bb5ab92019-01-11 11:11:10 +01004116TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4117 const int kFrameWidth = 320;
4118 const int kFrameHeight = 240;
4119 const int kFps = 30;
4120 const int kTargetBitrateBps = 120000;
4121 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4122
Erik Språng4c6ca302019-04-08 15:14:01 +02004123 video_stream_encoder_->OnBitrateUpdated(
4124 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004125
4126 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4127 max_framerate_ = kFps;
4128
4129 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4130 fake_encoder_.SimulateOvershoot(1.0);
4131 int num_dropped = 0;
4132 for (int i = 0; i < kNumFramesInRun; ++i) {
4133 video_source_.IncomingCapturedFrame(
4134 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4135 // Wait up to two frame durations for a frame to arrive.
4136 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4137 ++num_dropped;
4138 }
4139 timestamp_ms += 1000 / kFps;
4140 }
4141
Erik Språnga8d48ab2019-02-08 14:17:40 +01004142 // Framerate should be measured to be near the expected target rate.
4143 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4144
4145 // Frame drops should be within 5% of expected 0%.
4146 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004147
4148 // Make encoder produce frames at double the expected bitrate during 3 seconds
4149 // of video, verify number of drops. Rate needs to be slightly changed in
4150 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004151 double overshoot_factor = 2.0;
4152 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4153 // With bitrate adjuster, when need to overshoot even more to trigger
4154 // frame dropping.
4155 overshoot_factor *= 2;
4156 }
4157 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004158 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004159 DataRate::bps(kTargetBitrateBps + 1000),
4160 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004161 num_dropped = 0;
4162 for (int i = 0; i < kNumFramesInRun; ++i) {
4163 video_source_.IncomingCapturedFrame(
4164 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4165 // Wait up to two frame durations for a frame to arrive.
4166 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4167 ++num_dropped;
4168 }
4169 timestamp_ms += 1000 / kFps;
4170 }
4171
Erik Språng4c6ca302019-04-08 15:14:01 +02004172 video_stream_encoder_->OnBitrateUpdated(
4173 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004174
4175 // Target framerate should be still be near the expected target, despite
4176 // the frame drops.
4177 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4178
4179 // Frame drops should be within 5% of expected 50%.
4180 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004181
4182 video_stream_encoder_->Stop();
4183}
4184
4185TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4186 const int kFrameWidth = 320;
4187 const int kFrameHeight = 240;
4188 const int kActualInputFps = 24;
4189 const int kTargetBitrateBps = 120000;
4190
4191 ASSERT_GT(max_framerate_, kActualInputFps);
4192
4193 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4194 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004195 video_stream_encoder_->OnBitrateUpdated(
4196 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004197
4198 // Insert 3 seconds of video, with an input fps lower than configured max.
4199 for (int i = 0; i < kActualInputFps * 3; ++i) {
4200 video_source_.IncomingCapturedFrame(
4201 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4202 // Wait up to two frame durations for a frame to arrive.
4203 WaitForEncodedFrame(timestamp_ms);
4204 timestamp_ms += 1000 / kActualInputFps;
4205 }
4206
4207 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4208
4209 video_stream_encoder_->Stop();
4210}
4211
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004212TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4213 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004214 video_stream_encoder_->OnBitrateUpdated(
4215 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004216
4217 fake_encoder_.BlockNextEncode();
4218 video_source_.IncomingCapturedFrame(
4219 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4220 WaitForEncodedFrame(1);
4221 // On the very first frame full update should be forced.
4222 rect = fake_encoder_.GetLastUpdateRect();
4223 EXPECT_EQ(rect.offset_x, 0);
4224 EXPECT_EQ(rect.offset_y, 0);
4225 EXPECT_EQ(rect.height, codec_height_);
4226 EXPECT_EQ(rect.width, codec_width_);
4227 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4228 // call to ContinueEncode.
4229 video_source_.IncomingCapturedFrame(
4230 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4231 ExpectDroppedFrame();
4232 video_source_.IncomingCapturedFrame(
4233 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4234 ExpectDroppedFrame();
4235 fake_encoder_.ContinueEncode();
4236 WaitForEncodedFrame(3);
4237 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4238 rect = fake_encoder_.GetLastUpdateRect();
4239 EXPECT_EQ(rect.offset_x, 1);
4240 EXPECT_EQ(rect.offset_y, 0);
4241 EXPECT_EQ(rect.width, 10);
4242 EXPECT_EQ(rect.height, 1);
4243
4244 video_source_.IncomingCapturedFrame(
4245 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4246 WaitForEncodedFrame(4);
4247 // Previous frame was encoded, so no accumulation should happen.
4248 rect = fake_encoder_.GetLastUpdateRect();
4249 EXPECT_EQ(rect.offset_x, 0);
4250 EXPECT_EQ(rect.offset_y, 0);
4251 EXPECT_EQ(rect.width, 1);
4252 EXPECT_EQ(rect.height, 1);
4253
4254 video_stream_encoder_->Stop();
4255}
4256
Erik Språngd7329ca2019-02-21 21:19:53 +01004257TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004258 video_stream_encoder_->OnBitrateUpdated(
4259 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004260
4261 // First frame is always keyframe.
4262 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4263 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004264 EXPECT_THAT(
4265 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004266 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004267
4268 // Insert delta frame.
4269 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4270 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01004271 EXPECT_THAT(
4272 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004273 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004274
4275 // Request next frame be a key-frame.
4276 video_stream_encoder_->SendKeyFrame();
4277 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4278 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01004279 EXPECT_THAT(
4280 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004281 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004282
4283 video_stream_encoder_->Stop();
4284}
4285
4286TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4287 // Setup simulcast with three streams.
4288 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01004289 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004290 DataRate::bps(kSimulcastTargetBitrateBps),
4291 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004292 // Wait for all three layers before triggering event.
4293 sink_.SetNumExpectedLayers(3);
4294
4295 // First frame is always keyframe.
4296 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4297 WaitForEncodedFrame(1);
4298 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004299 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4300 VideoFrameType::kVideoFrameKey,
4301 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004302
4303 // Insert delta frame.
4304 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4305 WaitForEncodedFrame(2);
4306 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004307 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4308 VideoFrameType::kVideoFrameDelta,
4309 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004310
4311 // Request next frame be a key-frame.
4312 // Only first stream is configured to produce key-frame.
4313 video_stream_encoder_->SendKeyFrame();
4314 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4315 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02004316
4317 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4318 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01004319 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004320 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02004321 VideoFrameType::kVideoFrameKey,
4322 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004323
4324 video_stream_encoder_->Stop();
4325}
4326
4327TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
4328 // Configure internal source factory and setup test again.
4329 encoder_factory_.SetHasInternalSource(true);
4330 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004331 video_stream_encoder_->OnBitrateUpdated(
4332 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004333
4334 // Call encoder directly, simulating internal source where encoded frame
4335 // callback in VideoStreamEncoder is called despite no OnFrame().
4336 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
4337 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004338 EXPECT_THAT(
4339 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004340 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004341
Niels Möller8f7ce222019-03-21 15:43:58 +01004342 const std::vector<VideoFrameType> kDeltaFrame = {
4343 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01004344 // Need to set timestamp manually since manually for injected frame.
4345 VideoFrame frame = CreateFrame(101, nullptr);
4346 frame.set_timestamp(101);
4347 fake_encoder_.InjectFrame(frame, false);
4348 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004349 EXPECT_THAT(
4350 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004351 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004352
4353 // Request key-frame. The forces a dummy frame down into the encoder.
4354 fake_encoder_.ExpectNullFrame();
4355 video_stream_encoder_->SendKeyFrame();
4356 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004357 EXPECT_THAT(
4358 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004359 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004360
4361 video_stream_encoder_->Stop();
4362}
Erik Språngb7cb7b52019-02-26 15:52:33 +01004363
4364TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
4365 // Configure internal source factory and setup test again.
4366 encoder_factory_.SetHasInternalSource(true);
4367 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004368 video_stream_encoder_->OnBitrateUpdated(
4369 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01004370
4371 int64_t timestamp = 1;
4372 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02004373 image.SetEncodedData(
4374 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01004375 image.capture_time_ms_ = ++timestamp;
4376 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
4377 const int64_t kEncodeFinishDelayMs = 10;
4378 image.timing_.encode_start_ms = timestamp;
4379 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
4380 fake_encoder_.InjectEncodedImage(image);
4381 // Wait for frame without incrementing clock.
4382 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4383 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
4384 // capture timestamp should be kEncodeFinishDelayMs in the past.
4385 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
4386 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
4387 kEncodeFinishDelayMs);
4388
4389 video_stream_encoder_->Stop();
4390}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02004391
4392TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
4393 // Configure internal source factory and setup test again.
4394 encoder_factory_.SetHasInternalSource(true);
4395 ResetEncoder("H264", 1, 1, 1, false);
4396
4397 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
4398 image._frameType = VideoFrameType::kVideoFrameKey;
4399
4400 CodecSpecificInfo codec_specific_info;
4401 codec_specific_info.codecType = kVideoCodecH264;
4402
4403 RTPFragmentationHeader fragmentation;
4404 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4405 fragmentation.fragmentationOffset[0] = 4;
4406 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
4407
4408 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4409 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4410
4411 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4412 testing::ElementsAreArray(optimal_sps));
4413 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4414 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4415 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4416 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4417
4418 video_stream_encoder_->Stop();
4419}
4420
4421TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
4422 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
4423 0x00, 0x00, 0x03, 0x03, 0xF4,
4424 0x05, 0x03, 0xC7, 0xC0};
4425
4426 // Configure internal source factory and setup test again.
4427 encoder_factory_.SetHasInternalSource(true);
4428 ResetEncoder("H264", 1, 1, 1, false);
4429
4430 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
4431 image._frameType = VideoFrameType::kVideoFrameKey;
4432
4433 CodecSpecificInfo codec_specific_info;
4434 codec_specific_info.codecType = kVideoCodecH264;
4435
4436 RTPFragmentationHeader fragmentation;
4437 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4438 fragmentation.fragmentationOffset[0] = 4;
4439 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
4440
4441 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4442 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4443
4444 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4445 testing::ElementsAreArray(optimal_sps));
4446 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4447 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4448 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4449 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4450
4451 video_stream_encoder_->Stop();
4452}
4453
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02004454TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
4455 const int kFrameWidth = 1280;
4456 const int kFrameHeight = 720;
4457 const int kTargetBitrateBps = 300000; // To low for HD resolution.
4458
4459 video_stream_encoder_->OnBitrateUpdated(
4460 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
4461 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4462
4463 // Insert a first video frame. It should be dropped because of downscale in
4464 // resolution.
4465 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4466 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4467 frame.set_rotation(kVideoRotation_270);
4468 video_source_.IncomingCapturedFrame(frame);
4469
4470 ExpectDroppedFrame();
4471
4472 // Second frame is downscaled.
4473 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4474 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4475 frame.set_rotation(kVideoRotation_90);
4476 video_source_.IncomingCapturedFrame(frame);
4477
4478 WaitForEncodedFrame(timestamp_ms);
4479 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
4480
4481 // Insert another frame, also downscaled.
4482 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4483 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4484 frame.set_rotation(kVideoRotation_180);
4485 video_source_.IncomingCapturedFrame(frame);
4486
4487 WaitForEncodedFrame(timestamp_ms);
4488 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
4489
4490 video_stream_encoder_->Stop();
4491}
4492
perkj26091b12016-09-01 01:17:40 -07004493} // namespace webrtc