blob: 6f19edcbb1dc6b1235c1c8803b423f1f2dbae3b5 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080021#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010023#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020024#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020025#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010026#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020027#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070028#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080029#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020030#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010032#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010036#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020037#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "system_wrappers/include/sleep.h"
39#include "test/encoder_settings.h"
40#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020041#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "test/frame_generator.h"
43#include "test/gmock.h"
44#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020045#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070047
48namespace webrtc {
49
sprangb1ca0732017-02-01 08:38:12 -080050using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080051using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020052using ::testing::AllOf;
53using ::testing::Field;
54using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080055
perkj803d97f2016-11-01 11:45:46 -070056namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020057const int kMinPixelsPerFrame = 320 * 180;
58const int kMinFramerateFps = 2;
59const int kMinBalancedFramerateFps = 7;
60const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080061const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010062const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020063const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010064const uint32_t kSimulcastTargetBitrateBps = 3150000;
65const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080066const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070067const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020068const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020069const int64_t kProcessIntervalMs = 1000;
asapersson5f7226f2016-11-25 04:37:00 -080070
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020071uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
72 0x00, 0x00, 0x03, 0x03, 0xF4,
73 0x05, 0x03, 0xC7, 0xE0, 0x1B,
74 0x41, 0x10, 0x8D, 0x00};
75
perkj803d97f2016-11-01 11:45:46 -070076class TestBuffer : public webrtc::I420Buffer {
77 public:
78 TestBuffer(rtc::Event* event, int width, int height)
79 : I420Buffer(width, height), event_(event) {}
80
81 private:
82 friend class rtc::RefCountedObject<TestBuffer>;
83 ~TestBuffer() override {
84 if (event_)
85 event_->Set();
86 }
87 rtc::Event* const event_;
88};
89
Noah Richards51db4212019-06-12 06:59:12 -070090// A fake native buffer that can't be converted to I420.
91class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
92 public:
93 FakeNativeBuffer(rtc::Event* event, int width, int height)
94 : event_(event), width_(width), height_(height) {}
95 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
96 int width() const override { return width_; }
97 int height() const override { return height_; }
98 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
99 return nullptr;
100 }
101
102 private:
103 friend class rtc::RefCountedObject<FakeNativeBuffer>;
104 ~FakeNativeBuffer() override {
105 if (event_)
106 event_->Set();
107 }
108 rtc::Event* const event_;
109 const int width_;
110 const int height_;
111};
112
Niels Möller7dc26b72017-12-06 10:27:48 +0100113class CpuOveruseDetectorProxy : public OveruseFrameDetector {
114 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200115 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
116 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100117 last_target_framerate_fps_(-1) {}
118 virtual ~CpuOveruseDetectorProxy() {}
119
120 void OnTargetFramerateUpdated(int framerate_fps) override {
121 rtc::CritScope cs(&lock_);
122 last_target_framerate_fps_ = framerate_fps;
123 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
124 }
125
126 int GetLastTargetFramerate() {
127 rtc::CritScope cs(&lock_);
128 return last_target_framerate_fps_;
129 }
130
Niels Möller4db138e2018-04-19 09:04:13 +0200131 CpuOveruseOptions GetOptions() { return options_; }
132
Niels Möller7dc26b72017-12-06 10:27:48 +0100133 private:
134 rtc::CriticalSection lock_;
135 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
136};
137
mflodmancc3d4422017-08-03 08:27:51 -0700138class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700139 public:
Niels Möller213618e2018-07-24 09:29:58 +0200140 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200141 const VideoStreamEncoderSettings& settings,
142 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100143 : VideoStreamEncoder(Clock::GetRealTimeClock(),
144 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200145 stats_proxy,
146 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200147 std::unique_ptr<OveruseFrameDetector>(
148 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100149 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200150 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700151
sprangb1ca0732017-02-01 08:38:12 -0800152 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100153 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800154 encoder_queue()->PostTask([this, &event, reason, down] {
Åsa Perssonf5e5d252019-08-16 17:24:59 +0200155 if (down)
156 AdaptDown(reason);
157 else
158 AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700159 event.Set();
160 });
perkj070ba852017-02-16 15:46:27 -0800161 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700162 }
163
kthelgason2fc52542017-03-03 00:24:41 -0800164 // This is used as a synchronisation mechanism, to make sure that the
165 // encoder queue is not blocked before we start sending it frames.
166 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100167 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200168 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800169 ASSERT_TRUE(event.Wait(5000));
170 }
171
sprangb1ca0732017-02-01 08:38:12 -0800172 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800173
sprangb1ca0732017-02-01 08:38:12 -0800174 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800175
sprangb1ca0732017-02-01 08:38:12 -0800176 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800177
sprangb1ca0732017-02-01 08:38:12 -0800178 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700179
Niels Möller7dc26b72017-12-06 10:27:48 +0100180 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700181};
182
asapersson5f7226f2016-11-25 04:37:00 -0800183class VideoStreamFactory
184 : public VideoEncoderConfig::VideoStreamFactoryInterface {
185 public:
sprangfda496a2017-06-15 04:21:07 -0700186 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
187 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800188 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700189 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800190 }
191
192 private:
193 std::vector<VideoStream> CreateEncoderStreams(
194 int width,
195 int height,
196 const VideoEncoderConfig& encoder_config) override {
197 std::vector<VideoStream> streams =
198 test::CreateVideoStreams(width, height, encoder_config);
199 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100200 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700201 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800202 }
203 return streams;
204 }
sprangfda496a2017-06-15 04:21:07 -0700205
asapersson5f7226f2016-11-25 04:37:00 -0800206 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700207 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800208};
209
Noah Richards51db4212019-06-12 06:59:12 -0700210// Simulates simulcast behavior and makes highest stream resolutions divisible
211// by 4.
212class CroppingVideoStreamFactory
213 : public VideoEncoderConfig::VideoStreamFactoryInterface {
214 public:
215 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
216 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
217 EXPECT_GT(num_temporal_layers, 0u);
218 EXPECT_GT(framerate, 0);
219 }
220
221 private:
222 std::vector<VideoStream> CreateEncoderStreams(
223 int width,
224 int height,
225 const VideoEncoderConfig& encoder_config) override {
226 std::vector<VideoStream> streams = test::CreateVideoStreams(
227 width - width % 4, height - height % 4, encoder_config);
228 for (VideoStream& stream : streams) {
229 stream.num_temporal_layers = num_temporal_layers_;
230 stream.max_framerate = framerate_;
231 }
232 return streams;
233 }
234
235 const size_t num_temporal_layers_;
236 const int framerate_;
237};
238
sprangb1ca0732017-02-01 08:38:12 -0800239class AdaptingFrameForwarder : public test::FrameForwarder {
240 public:
241 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700242 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800243
244 void set_adaptation_enabled(bool enabled) {
245 rtc::CritScope cs(&crit_);
246 adaptation_enabled_ = enabled;
247 }
248
asaperssonfab67072017-04-04 05:51:49 -0700249 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800250 rtc::CritScope cs(&crit_);
251 return adaptation_enabled_;
252 }
253
asapersson09f05612017-05-15 23:40:18 -0700254 rtc::VideoSinkWants last_wants() const {
255 rtc::CritScope cs(&crit_);
256 return last_wants_;
257 }
258
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200259 absl::optional<int> last_sent_width() const { return last_width_; }
260 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800261
sprangb1ca0732017-02-01 08:38:12 -0800262 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
263 int cropped_width = 0;
264 int cropped_height = 0;
265 int out_width = 0;
266 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700267 if (adaption_enabled()) {
268 if (adapter_.AdaptFrameResolution(
269 video_frame.width(), video_frame.height(),
270 video_frame.timestamp_us() * 1000, &cropped_width,
271 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100272 VideoFrame adapted_frame =
273 VideoFrame::Builder()
274 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
275 nullptr, out_width, out_height))
276 .set_timestamp_rtp(99)
277 .set_timestamp_ms(99)
278 .set_rotation(kVideoRotation_0)
279 .build();
sprangc5d62e22017-04-02 23:53:04 -0700280 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
281 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800282 last_width_.emplace(adapted_frame.width());
283 last_height_.emplace(adapted_frame.height());
284 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200285 last_width_ = absl::nullopt;
286 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700287 }
sprangb1ca0732017-02-01 08:38:12 -0800288 } else {
289 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800290 last_width_.emplace(video_frame.width());
291 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800292 }
293 }
294
295 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
296 const rtc::VideoSinkWants& wants) override {
297 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700298 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700299 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
300 wants.max_pixel_count,
301 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800302 test::FrameForwarder::AddOrUpdateSink(sink, wants);
303 }
sprangb1ca0732017-02-01 08:38:12 -0800304 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700305 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
306 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200307 absl::optional<int> last_width_;
308 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800309};
sprangc5d62e22017-04-02 23:53:04 -0700310
Niels Möller213618e2018-07-24 09:29:58 +0200311// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700312class MockableSendStatisticsProxy : public SendStatisticsProxy {
313 public:
314 MockableSendStatisticsProxy(Clock* clock,
315 const VideoSendStream::Config& config,
316 VideoEncoderConfig::ContentType content_type)
317 : SendStatisticsProxy(clock, config, content_type) {}
318
319 VideoSendStream::Stats GetStats() override {
320 rtc::CritScope cs(&lock_);
321 if (mock_stats_)
322 return *mock_stats_;
323 return SendStatisticsProxy::GetStats();
324 }
325
Niels Möller213618e2018-07-24 09:29:58 +0200326 int GetInputFrameRate() const override {
327 rtc::CritScope cs(&lock_);
328 if (mock_stats_)
329 return mock_stats_->input_frame_rate;
330 return SendStatisticsProxy::GetInputFrameRate();
331 }
sprangc5d62e22017-04-02 23:53:04 -0700332 void SetMockStats(const VideoSendStream::Stats& stats) {
333 rtc::CritScope cs(&lock_);
334 mock_stats_.emplace(stats);
335 }
336
337 void ResetMockStats() {
338 rtc::CritScope cs(&lock_);
339 mock_stats_.reset();
340 }
341
342 private:
343 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200344 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700345};
346
sprang4847ae62017-06-27 07:06:52 -0700347class MockBitrateObserver : public VideoBitrateAllocationObserver {
348 public:
Erik Språng566124a2018-04-23 12:32:22 +0200349 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700350};
351
perkj803d97f2016-11-01 11:45:46 -0700352} // namespace
353
mflodmancc3d4422017-08-03 08:27:51 -0700354class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700355 public:
356 static const int kDefaultTimeoutMs = 30 * 1000;
357
mflodmancc3d4422017-08-03 08:27:51 -0700358 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700359 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700360 codec_width_(320),
361 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200362 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200363 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700364 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200365 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700366 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700367 Clock::GetRealTimeClock(),
368 video_send_config_,
369 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700370 sink_(&fake_encoder_) {}
371
372 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700373 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700374 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200375 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800376 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200377 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200378 video_send_config_.rtp.payload_name = "FAKE";
379 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700380
Per512ecb32016-09-23 15:52:06 +0200381 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200382 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700383 video_encoder_config.video_stream_factory =
384 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100385 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700386
387 // Framerate limit is specified by the VideoStreamFactory.
388 std::vector<VideoStream> streams =
389 video_encoder_config.video_stream_factory->CreateEncoderStreams(
390 codec_width_, codec_height_, video_encoder_config);
391 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200392 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700393
Niels Möllerf1338562018-04-26 09:51:47 +0200394 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800395 }
396
Niels Möllerf1338562018-04-26 09:51:47 +0200397 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700398 if (video_stream_encoder_)
399 video_stream_encoder_->Stop();
400 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200401 stats_proxy_.get(), video_send_config_.encoder_settings,
402 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700403 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700405 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700406 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
407 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200408 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700409 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800410 }
411
412 void ResetEncoder(const std::string& payload_name,
413 size_t num_streams,
414 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700415 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700416 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200417 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800418
419 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200420 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800421 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100422 video_encoder_config.max_bitrate_bps =
423 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800424 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700425 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
426 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700427 video_encoder_config.content_type =
428 screenshare ? VideoEncoderConfig::ContentType::kScreen
429 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700430 if (payload_name == "VP9") {
431 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
432 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
433 video_encoder_config.encoder_specific_settings =
434 new rtc::RefCountedObject<
435 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
436 }
Niels Möllerf1338562018-04-26 09:51:47 +0200437 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700438 }
439
sprang57c2fff2017-01-16 06:24:02 -0800440 VideoFrame CreateFrame(int64_t ntp_time_ms,
441 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100442 VideoFrame frame =
443 VideoFrame::Builder()
444 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
445 destruction_event, codec_width_, codec_height_))
446 .set_timestamp_rtp(99)
447 .set_timestamp_ms(99)
448 .set_rotation(kVideoRotation_0)
449 .build();
sprang57c2fff2017-01-16 06:24:02 -0800450 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700451 return frame;
452 }
453
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100454 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
455 rtc::Event* destruction_event,
456 int offset_x) const {
457 VideoFrame frame =
458 VideoFrame::Builder()
459 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
460 destruction_event, codec_width_, codec_height_))
461 .set_timestamp_rtp(99)
462 .set_timestamp_ms(99)
463 .set_rotation(kVideoRotation_0)
464 .set_update_rect({offset_x, 0, 1, 1})
465 .build();
466 frame.set_ntp_time_ms(ntp_time_ms);
467 return frame;
468 }
469
sprang57c2fff2017-01-16 06:24:02 -0800470 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100471 VideoFrame frame =
472 VideoFrame::Builder()
473 .set_video_frame_buffer(
474 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
475 .set_timestamp_rtp(99)
476 .set_timestamp_ms(99)
477 .set_rotation(kVideoRotation_0)
478 .build();
sprang57c2fff2017-01-16 06:24:02 -0800479 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700480 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700481 return frame;
482 }
483
Noah Richards51db4212019-06-12 06:59:12 -0700484 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
485 rtc::Event* destruction_event,
486 int width,
487 int height) const {
488 VideoFrame frame =
489 VideoFrame::Builder()
490 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
491 destruction_event, width, height))
492 .set_timestamp_rtp(99)
493 .set_timestamp_ms(99)
494 .set_rotation(kVideoRotation_0)
495 .build();
496 frame.set_ntp_time_ms(ntp_time_ms);
497 return frame;
498 }
499
500 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
501 rtc::Event* destruction_event) const {
502 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
503 codec_height_);
504 }
505
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100506 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
507 MockBitrateObserver bitrate_observer;
508 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
509
510 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
511 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200512 video_stream_encoder_->OnBitrateUpdated(
513 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
514 DataRate::bps(kTargetBitrateBps), 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100515
516 video_source_.IncomingCapturedFrame(
517 CreateFrame(1, codec_width_, codec_height_));
518 WaitForEncodedFrame(1);
519 }
520
asapersson02465b82017-04-10 01:12:52 -0700521 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700522 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700523 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
524 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700525 }
526
asapersson09f05612017-05-15 23:40:18 -0700527 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
528 const rtc::VideoSinkWants& wants2) {
529 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
530 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
531 }
532
Åsa Persson8c1bf952018-09-13 10:42:19 +0200533 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
534 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
535 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
536 EXPECT_FALSE(wants.target_pixel_count);
537 }
538
asapersson09f05612017-05-15 23:40:18 -0700539 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
540 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200541 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700542 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
543 EXPECT_GT(wants1.max_pixel_count, 0);
544 }
545
546 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
547 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200548 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700549 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
550 }
551
asaperssonf7e294d2017-06-13 23:25:22 -0700552 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
553 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200554 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700555 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
556 }
557
558 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
559 const rtc::VideoSinkWants& wants2) {
560 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
561 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
562 }
563
564 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
565 const rtc::VideoSinkWants& wants2) {
566 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
567 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
568 }
569
570 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
571 const rtc::VideoSinkWants& wants2) {
572 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
573 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
574 EXPECT_GT(wants1.max_pixel_count, 0);
575 }
576
577 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
578 const rtc::VideoSinkWants& wants2) {
579 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
580 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
581 }
582
asapersson09f05612017-05-15 23:40:18 -0700583 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
584 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200585 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700586 EXPECT_LT(wants.max_pixel_count, pixel_count);
587 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700588 }
589
590 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
591 EXPECT_LT(wants.max_framerate_fps, fps);
592 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
593 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700594 }
595
asaperssonf7e294d2017-06-13 23:25:22 -0700596 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
597 int expected_fps) {
598 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
599 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
600 EXPECT_FALSE(wants.target_pixel_count);
601 }
602
Jonathan Yubc771b72017-12-08 17:04:29 -0800603 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
604 int last_frame_pixels) {
605 // Balanced mode should always scale FPS to the desired range before
606 // attempting to scale resolution.
607 int fps_limit = wants.max_framerate_fps;
608 if (last_frame_pixels <= 320 * 240) {
609 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
610 } else if (last_frame_pixels <= 480 * 270) {
611 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
612 } else if (last_frame_pixels <= 640 * 480) {
613 EXPECT_LE(15, fps_limit);
614 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200615 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800616 }
617 }
618
sprang4847ae62017-06-27 07:06:52 -0700619 void WaitForEncodedFrame(int64_t expected_ntp_time) {
620 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200621 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700622 }
623
624 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
625 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200626 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700627 return ok;
628 }
629
630 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
631 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200632 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700633 }
634
635 void ExpectDroppedFrame() {
636 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200637 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700638 }
639
640 bool WaitForFrame(int64_t timeout_ms) {
641 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200642 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700643 return ok;
644 }
645
perkj26091b12016-09-01 01:17:40 -0700646 class TestEncoder : public test::FakeEncoder {
647 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100648 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700649
asaperssonfab67072017-04-04 05:51:49 -0700650 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800651 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700652 return config_;
653 }
654
655 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800656 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700657 block_next_encode_ = true;
658 }
659
Erik Språngaed30702018-11-05 12:57:17 +0100660 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800661 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100662 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100663 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100664 if (quality_scaling_) {
665 info.scaling_settings =
666 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
667 }
668 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100669 for (int i = 0; i < kMaxSpatialLayers; ++i) {
670 if (temporal_layers_supported_[i]) {
671 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
672 info.fps_allocation[i].resize(num_layers);
673 }
674 }
Erik Språngaed30702018-11-05 12:57:17 +0100675 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200676
677 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Erik Språngaed30702018-11-05 12:57:17 +0100678 return info;
kthelgason876222f2016-11-29 01:44:11 -0800679 }
680
Erik Språngb7cb7b52019-02-26 15:52:33 +0100681 int32_t RegisterEncodeCompleteCallback(
682 EncodedImageCallback* callback) override {
683 rtc::CritScope lock(&local_crit_sect_);
684 encoded_image_callback_ = callback;
685 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
686 }
687
perkjfa10b552016-10-02 23:45:26 -0700688 void ContinueEncode() { continue_encode_event_.Set(); }
689
690 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
691 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800692 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700693 EXPECT_EQ(timestamp_, timestamp);
694 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
695 }
696
kthelgason2fc52542017-03-03 00:24:41 -0800697 void SetQualityScaling(bool b) {
698 rtc::CritScope lock(&local_crit_sect_);
699 quality_scaling_ = b;
700 }
kthelgasonad9010c2017-02-14 00:46:51 -0800701
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100702 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
703 rtc::CritScope lock(&local_crit_sect_);
704 is_hardware_accelerated_ = is_hardware_accelerated;
705 }
706
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100707 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
708 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
709 rtc::CritScope lock(&local_crit_sect_);
710 temporal_layers_supported_[spatial_idx] = supported;
711 }
712
Sergey Silkin6456e352019-07-08 17:56:40 +0200713 void SetResolutionBitrateLimits(
714 std::vector<ResolutionBitrateLimits> thresholds) {
715 rtc::CritScope cs(&local_crit_sect_);
716 resolution_bitrate_limits_ = thresholds;
717 }
718
sprangfe627f32017-03-29 08:24:59 -0700719 void ForceInitEncodeFailure(bool force_failure) {
720 rtc::CritScope lock(&local_crit_sect_);
721 force_init_encode_failed_ = force_failure;
722 }
723
Niels Möller6bb5ab92019-01-11 11:11:10 +0100724 void SimulateOvershoot(double rate_factor) {
725 rtc::CritScope lock(&local_crit_sect_);
726 rate_factor_ = rate_factor;
727 }
728
Erik Språngd7329ca2019-02-21 21:19:53 +0100729 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100730 rtc::CritScope lock(&local_crit_sect_);
731 return last_framerate_;
732 }
733
Erik Språngd7329ca2019-02-21 21:19:53 +0100734 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100735 rtc::CritScope lock(&local_crit_sect_);
736 return last_update_rect_;
737 }
738
Niels Möller87e2d782019-03-07 10:18:23 +0100739 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100740 rtc::CritScope lock(&local_crit_sect_);
741 return last_frame_types_;
742 }
743
744 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100745 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100746 keyframe ? VideoFrameType::kVideoFrameKey
747 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100748 {
749 rtc::CritScope lock(&local_crit_sect_);
750 last_frame_types_ = frame_type;
751 }
Niels Möllerb859b322019-03-07 12:40:01 +0100752 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100753 }
754
Erik Språngb7cb7b52019-02-26 15:52:33 +0100755 void InjectEncodedImage(const EncodedImage& image) {
756 rtc::CritScope lock(&local_crit_sect_);
757 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
758 }
759
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200760 void InjectEncodedImage(const EncodedImage& image,
761 const CodecSpecificInfo* codec_specific_info,
762 const RTPFragmentationHeader* fragmentation) {
763 rtc::CritScope lock(&local_crit_sect_);
764 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
765 fragmentation);
766 }
767
Erik Språngd7329ca2019-02-21 21:19:53 +0100768 void ExpectNullFrame() {
769 rtc::CritScope lock(&local_crit_sect_);
770 expect_null_frame_ = true;
771 }
772
Erik Språng5056af02019-09-02 15:53:11 +0200773 absl::optional<VideoEncoder::RateControlParameters>
774 GetAndResetLastRateControlSettings() {
775 auto settings = last_rate_control_settings_;
776 last_rate_control_settings_.reset();
777 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100778 }
779
Sergey Silkin5ee69672019-07-02 14:18:34 +0200780 int GetNumEncoderInitializations() const {
781 rtc::CritScope lock(&local_crit_sect_);
782 return num_encoder_initializations_;
783 }
784
perkjfa10b552016-10-02 23:45:26 -0700785 private:
perkj26091b12016-09-01 01:17:40 -0700786 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100787 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700788 bool block_encode;
789 {
brandtre78d2662017-01-16 05:57:16 -0800790 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100791 if (expect_null_frame_) {
792 EXPECT_EQ(input_image.timestamp(), 0u);
793 EXPECT_EQ(input_image.width(), 1);
794 last_frame_types_ = *frame_types;
795 expect_null_frame_ = false;
796 } else {
797 EXPECT_GT(input_image.timestamp(), timestamp_);
798 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
799 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
800 }
perkj26091b12016-09-01 01:17:40 -0700801
802 timestamp_ = input_image.timestamp();
803 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700804 last_input_width_ = input_image.width();
805 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700806 block_encode = block_next_encode_;
807 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100808 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100809 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700810 }
Niels Möllerb859b322019-03-07 12:40:01 +0100811 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700812 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700813 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700814 return result;
815 }
816
sprangfe627f32017-03-29 08:24:59 -0700817 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200818 const Settings& settings) override {
819 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200820
sprangfe627f32017-03-29 08:24:59 -0700821 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100822 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200823
824 ++num_encoder_initializations_;
825
Erik Språng82fad3d2018-03-21 09:57:23 +0100826 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700827 // Simulate setting up temporal layers, in order to validate the life
828 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100829 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200830 frame_buffer_controller_ =
831 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700832 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100833 if (force_init_encode_failed_) {
834 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700835 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100836 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100837
Erik Språngb7cb7b52019-02-26 15:52:33 +0100838 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700839 return res;
840 }
841
Erik Språngb7cb7b52019-02-26 15:52:33 +0100842 int32_t Release() override {
843 rtc::CritScope lock(&local_crit_sect_);
844 EXPECT_NE(initialized_, EncoderState::kUninitialized);
845 initialized_ = EncoderState::kUninitialized;
846 return FakeEncoder::Release();
847 }
848
Erik Språng16cb8f52019-04-12 13:59:09 +0200849 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100850 rtc::CritScope lock(&local_crit_sect_);
851 VideoBitrateAllocation adjusted_rate_allocation;
852 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
853 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200854 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100855 adjusted_rate_allocation.SetBitrate(
856 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200857 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100858 rate_factor_));
859 }
860 }
861 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200862 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200863 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200864 RateControlParameters adjusted_paramters = parameters;
865 adjusted_paramters.bitrate = adjusted_rate_allocation;
866 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100867 }
868
brandtre78d2662017-01-16 05:57:16 -0800869 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100870 enum class EncoderState {
871 kUninitialized,
872 kInitializationFailed,
873 kInitialized
874 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
875 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700876 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700877 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700878 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
879 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
880 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
881 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
882 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100883 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100884 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700885 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100886 absl::optional<bool>
887 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
888 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700889 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100890 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
891 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200892 absl::optional<VideoEncoder::RateControlParameters>
893 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100894 VideoFrame::UpdateRect last_update_rect_
895 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100896 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100897 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100898 EncodedImageCallback* encoded_image_callback_
899 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 11:20:09 +0200900 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200901 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +0200902 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
903 RTC_GUARDED_BY(local_crit_sect_);
perkj26091b12016-09-01 01:17:40 -0700904 };
905
mflodmancc3d4422017-08-03 08:27:51 -0700906 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700907 public:
908 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100909 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700910
perkj26091b12016-09-01 01:17:40 -0700911 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700912 EXPECT_TRUE(
913 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
914 }
915
916 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
917 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700918 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700919 if (!encoded_frame_event_.Wait(timeout_ms))
920 return false;
perkj26091b12016-09-01 01:17:40 -0700921 {
922 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800923 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700924 }
925 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700926 return true;
perkj26091b12016-09-01 01:17:40 -0700927 }
928
sprangb1ca0732017-02-01 08:38:12 -0800929 void WaitForEncodedFrame(uint32_t expected_width,
930 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700931 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100932 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700933 }
934
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100935 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700936 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800937 uint32_t width = 0;
938 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800939 {
940 rtc::CritScope lock(&crit_);
941 width = last_width_;
942 height = last_height_;
943 }
944 EXPECT_EQ(expected_height, height);
945 EXPECT_EQ(expected_width, width);
946 }
947
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200948 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
949 VideoRotation rotation;
950 {
951 rtc::CritScope lock(&crit_);
952 rotation = last_rotation_;
953 }
954 EXPECT_EQ(expected_rotation, rotation);
955 }
956
kthelgason2fc52542017-03-03 00:24:41 -0800957 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800958
sprangc5d62e22017-04-02 23:53:04 -0700959 bool WaitForFrame(int64_t timeout_ms) {
960 return encoded_frame_event_.Wait(timeout_ms);
961 }
962
perkj26091b12016-09-01 01:17:40 -0700963 void SetExpectNoFrames() {
964 rtc::CritScope lock(&crit_);
965 expect_frames_ = false;
966 }
967
asaperssonfab67072017-04-04 05:51:49 -0700968 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200969 rtc::CritScope lock(&crit_);
970 return number_of_reconfigurations_;
971 }
972
asaperssonfab67072017-04-04 05:51:49 -0700973 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200974 rtc::CritScope lock(&crit_);
975 return min_transmit_bitrate_bps_;
976 }
977
Erik Språngd7329ca2019-02-21 21:19:53 +0100978 void SetNumExpectedLayers(size_t num_layers) {
979 rtc::CritScope lock(&crit_);
980 num_expected_layers_ = num_layers;
981 }
982
Erik Språngb7cb7b52019-02-26 15:52:33 +0100983 int64_t GetLastCaptureTimeMs() const {
984 rtc::CritScope lock(&crit_);
985 return last_capture_time_ms_;
986 }
987
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200988 std::vector<uint8_t> GetLastEncodedImageData() {
989 rtc::CritScope lock(&crit_);
990 return std::move(last_encoded_image_data_);
991 }
992
993 RTPFragmentationHeader GetLastFragmentation() {
994 rtc::CritScope lock(&crit_);
995 return std::move(last_fragmentation_);
996 }
997
perkj26091b12016-09-01 01:17:40 -0700998 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700999 Result OnEncodedImage(
1000 const EncodedImage& encoded_image,
1001 const CodecSpecificInfo* codec_specific_info,
1002 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001003 rtc::CritScope lock(&crit_);
1004 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001005 last_encoded_image_data_ = std::vector<uint8_t>(
1006 encoded_image.data(), encoded_image.data() + encoded_image.size());
1007 if (fragmentation) {
1008 last_fragmentation_.CopyFrom(*fragmentation);
1009 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001010 uint32_t timestamp = encoded_image.Timestamp();
1011 if (last_timestamp_ != timestamp) {
1012 num_received_layers_ = 1;
1013 } else {
1014 ++num_received_layers_;
1015 }
1016 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001017 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001018 last_width_ = encoded_image._encodedWidth;
1019 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001020 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001021 if (num_received_layers_ == num_expected_layers_) {
1022 encoded_frame_event_.Set();
1023 }
sprangb1ca0732017-02-01 08:38:12 -08001024 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001025 }
1026
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001027 void OnEncoderConfigurationChanged(
1028 std::vector<VideoStream> streams,
1029 VideoEncoderConfig::ContentType content_type,
1030 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001031 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001032 ++number_of_reconfigurations_;
1033 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1034 }
1035
perkj26091b12016-09-01 01:17:40 -07001036 rtc::CriticalSection crit_;
1037 TestEncoder* test_encoder_;
1038 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001039 std::vector<uint8_t> last_encoded_image_data_;
1040 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001041 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001042 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001043 uint32_t last_height_ = 0;
1044 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001045 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001046 size_t num_expected_layers_ = 1;
1047 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001048 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001049 int number_of_reconfigurations_ = 0;
1050 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001051 };
1052
Sergey Silkin5ee69672019-07-02 14:18:34 +02001053 class VideoBitrateAllocatorProxyFactory
1054 : public VideoBitrateAllocatorFactory {
1055 public:
1056 VideoBitrateAllocatorProxyFactory()
1057 : bitrate_allocator_factory_(
1058 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1059
1060 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1061 const VideoCodec& codec) override {
1062 rtc::CritScope lock(&crit_);
1063 codec_config_ = codec;
1064 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1065 }
1066
1067 VideoCodec codec_config() const {
1068 rtc::CritScope lock(&crit_);
1069 return codec_config_;
1070 }
1071
1072 private:
1073 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1074
1075 rtc::CriticalSection crit_;
1076 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1077 };
1078
perkj26091b12016-09-01 01:17:40 -07001079 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001080 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001081 int codec_width_;
1082 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001083 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001084 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001085 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001086 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001087 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001088 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001089 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001090 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001091 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001092 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001093};
1094
mflodmancc3d4422017-08-03 08:27:51 -07001095TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001096 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001097 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1098 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001099 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001100 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001101 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001102 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001103 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001104}
1105
mflodmancc3d4422017-08-03 08:27:51 -07001106TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001107 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001108 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001109 // The encoder will cache up to one frame for a short duration. Adding two
1110 // frames means that the first frame will be dropped and the second frame will
1111 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001112 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001113 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001114 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001115
Erik Språng4c6ca302019-04-08 15:14:01 +02001116 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001117 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1118 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001119
Sebastian Janssona3177052018-04-10 13:05:49 +02001120 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001121 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001122 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1123
1124 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001125 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001126}
1127
mflodmancc3d4422017-08-03 08:27:51 -07001128TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001129 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001130 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1131 DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001132 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001133 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001134
Florent Castellia8336d32019-09-09 13:36:55 +02001135 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
1136 DataRate::bps(0), 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001137 // The encoder will cache up to one frame for a short duration. Adding two
1138 // frames means that the first frame will be dropped and the second frame will
1139 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001140 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001141 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001142
Erik Språng4c6ca302019-04-08 15:14:01 +02001143 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001144 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1145 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001146 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001147 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1148 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001149 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001150}
1151
mflodmancc3d4422017-08-03 08:27:51 -07001152TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001153 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001154 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1155 DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001156 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001157 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001158
1159 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001160 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001161
perkja49cbd32016-09-16 07:53:41 -07001162 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001163 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001164 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001165}
1166
mflodmancc3d4422017-08-03 08:27:51 -07001167TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001168 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001169 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1170 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001171
perkja49cbd32016-09-16 07:53:41 -07001172 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001173 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001174
mflodmancc3d4422017-08-03 08:27:51 -07001175 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001176 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001177 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001178 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1179 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001180}
1181
mflodmancc3d4422017-08-03 08:27:51 -07001182TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001183 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001184 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1185 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001186
1187 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001188 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001189 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001190 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1191 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001192 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1193 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001194 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001195 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001196
mflodmancc3d4422017-08-03 08:27:51 -07001197 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001198}
1199
Noah Richards51db4212019-06-12 06:59:12 -07001200TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1201 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001202 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1203 DataRate::bps(kTargetBitrateBps), 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001204
1205 rtc::Event frame_destroyed_event;
1206 video_source_.IncomingCapturedFrame(
1207 CreateFakeNativeFrame(1, &frame_destroyed_event));
1208 ExpectDroppedFrame();
1209 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1210 video_stream_encoder_->Stop();
1211}
1212
1213TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1214 // Use the cropping factory.
1215 video_encoder_config_.video_stream_factory =
1216 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1217 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1218 kMaxPayloadLength);
1219 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1220
1221 // Capture a frame at codec_width_/codec_height_.
1222 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001223 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1224 DataRate::bps(kTargetBitrateBps), 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001225 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1226 WaitForEncodedFrame(1);
1227 // The encoder will have been configured once.
1228 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1229 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1230 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1231
1232 // Now send in a fake frame that needs to be cropped as the width/height
1233 // aren't divisible by 4 (see CreateEncoderStreams above).
1234 rtc::Event frame_destroyed_event;
1235 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1236 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1237 ExpectDroppedFrame();
1238 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1239 video_stream_encoder_->Stop();
1240}
1241
mflodmancc3d4422017-08-03 08:27:51 -07001242TEST_F(VideoStreamEncoderTest,
1243 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001244 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001245 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1246 DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001247 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001248
1249 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001250 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001251 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001252 // The encoder will have been configured once when the first frame is
1253 // received.
1254 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001255
1256 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001257 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001258 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001259 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001260 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001261
1262 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001263 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001264 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001265 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001266 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001267
mflodmancc3d4422017-08-03 08:27:51 -07001268 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001269}
1270
mflodmancc3d4422017-08-03 08:27:51 -07001271TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001272 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001273 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1274 DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001275
1276 // Capture a frame and wait for it to synchronize with the encoder thread.
1277 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001278 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001279 // The encoder will have been configured once.
1280 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001281 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1282 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1283
1284 codec_width_ *= 2;
1285 codec_height_ *= 2;
1286 // Capture a frame with a higher resolution and wait for it to synchronize
1287 // with the encoder thread.
1288 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001289 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001290 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1291 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001292 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001293
mflodmancc3d4422017-08-03 08:27:51 -07001294 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001295}
1296
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001297TEST_F(VideoStreamEncoderTest,
1298 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1299 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001300 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1301 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001302
1303 // Capture a frame and wait for it to synchronize with the encoder thread.
1304 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1305 WaitForEncodedFrame(1);
1306
1307 VideoEncoderConfig video_encoder_config;
1308 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1309 // Changing the max payload data length recreates encoder.
1310 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1311 kMaxPayloadLength / 2);
1312
1313 // Capture a frame and wait for it to synchronize with the encoder thread.
1314 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1315 WaitForEncodedFrame(2);
1316 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1317
1318 video_stream_encoder_->Stop();
1319}
1320
Sergey Silkin5ee69672019-07-02 14:18:34 +02001321TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1322 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001323 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1324 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001325
1326 VideoEncoderConfig video_encoder_config;
1327 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1328 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1329 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1330 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1331 kMaxPayloadLength);
1332
1333 // Capture a frame and wait for it to synchronize with the encoder thread.
1334 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1335 WaitForEncodedFrame(1);
1336 // The encoder will have been configured once when the first frame is
1337 // received.
1338 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1339 EXPECT_EQ(kTargetBitrateBps,
1340 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1341 EXPECT_EQ(kStartBitrateBps,
1342 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1343
Sergey Silkin6456e352019-07-08 17:56:40 +02001344 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1345 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001346 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1347 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1348 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1349 kMaxPayloadLength);
1350
1351 // Capture a frame and wait for it to synchronize with the encoder thread.
1352 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1353 WaitForEncodedFrame(2);
1354 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1355 // Bitrate limits have changed - rate allocator should be reconfigured,
1356 // encoder should not be reconfigured.
1357 EXPECT_EQ(kTargetBitrateBps * 2,
1358 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1359 EXPECT_EQ(kStartBitrateBps * 2,
1360 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1361 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1362
1363 video_stream_encoder_->Stop();
1364}
1365
Sergey Silkin6456e352019-07-08 17:56:40 +02001366TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001367 EncoderRecommendedBitrateLimitsDoNotOverrideAppBitrateLimits) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001368 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001369 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1370 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001371
Sergey Silkin6456e352019-07-08 17:56:40 +02001372 VideoEncoderConfig video_encoder_config;
1373 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1374 video_encoder_config.max_bitrate_bps = 0;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001375 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001376 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1377 kMaxPayloadLength);
1378
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001379 video_source_.IncomingCapturedFrame(CreateFrame(1, 360, 180));
Sergey Silkin6456e352019-07-08 17:56:40 +02001380 WaitForEncodedFrame(1);
Sergey Silkin6456e352019-07-08 17:56:40 +02001381
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001382 // Get the default bitrate limits and use them as baseline for custom
1383 // application and encoder recommended limits.
1384 const uint32_t kDefaultMinBitrateKbps =
1385 bitrate_allocator_factory_.codec_config().minBitrate;
1386 const uint32_t kDefaultMaxBitrateKbps =
1387 bitrate_allocator_factory_.codec_config().maxBitrate;
1388 const uint32_t kEncMinBitrateKbps = kDefaultMinBitrateKbps * 2;
1389 const uint32_t kEncMaxBitrateKbps = kDefaultMaxBitrateKbps * 2;
1390 const uint32_t kAppMinBitrateKbps = kDefaultMinBitrateKbps * 3;
1391 const uint32_t kAppMaxBitrateKbps = kDefaultMaxBitrateKbps * 3;
1392
1393 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1394 codec_width_ * codec_height_, kEncMinBitrateKbps * 1000,
1395 kEncMinBitrateKbps * 1000, kEncMaxBitrateKbps * 1000);
1396 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1397
1398 // Change resolution. This will trigger encoder re-configuration and video
1399 // stream encoder will pick up the bitrate limits recommended by encoder.
1400 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1401 WaitForEncodedFrame(2);
1402 video_source_.IncomingCapturedFrame(CreateFrame(3, 360, 180));
1403 WaitForEncodedFrame(3);
1404
1405 // App bitrate limits are not set - bitrate limits recommended by encoder
1406 // should be used.
1407 EXPECT_EQ(kEncMaxBitrateKbps,
1408 bitrate_allocator_factory_.codec_config().maxBitrate);
1409 EXPECT_EQ(kEncMinBitrateKbps,
1410 bitrate_allocator_factory_.codec_config().minBitrate);
1411
1412 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1413 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
1414 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1415 kMaxPayloadLength);
1416 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1417 WaitForEncodedFrame(4);
1418
1419 // App limited the max bitrate - bitrate limits recommended by encoder should
1420 // not be applied.
1421 EXPECT_EQ(kAppMaxBitrateKbps,
1422 bitrate_allocator_factory_.codec_config().maxBitrate);
1423 EXPECT_EQ(kDefaultMinBitrateKbps,
1424 bitrate_allocator_factory_.codec_config().minBitrate);
1425
1426 video_encoder_config.max_bitrate_bps = 0;
1427 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1428 kAppMinBitrateKbps * 1000;
1429 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1430 kMaxPayloadLength);
1431 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1432 WaitForEncodedFrame(5);
1433
1434 // App limited the min bitrate - bitrate limits recommended by encoder should
1435 // not be applied.
1436 EXPECT_EQ(kDefaultMaxBitrateKbps,
1437 bitrate_allocator_factory_.codec_config().maxBitrate);
1438 EXPECT_EQ(kAppMinBitrateKbps,
1439 bitrate_allocator_factory_.codec_config().minBitrate);
1440
1441 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1442 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1443 kAppMinBitrateKbps * 1000;
Sergey Silkin6456e352019-07-08 17:56:40 +02001444 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1445 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001446 video_source_.IncomingCapturedFrame(CreateFrame(6, nullptr));
1447 WaitForEncodedFrame(6);
Sergey Silkin6456e352019-07-08 17:56:40 +02001448
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001449 // App limited both min and max bitrates - bitrate limits recommended by
1450 // encoder should not be applied.
1451 EXPECT_EQ(kAppMaxBitrateKbps,
1452 bitrate_allocator_factory_.codec_config().maxBitrate);
1453 EXPECT_EQ(kAppMinBitrateKbps,
1454 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001455
1456 video_stream_encoder_->Stop();
1457}
1458
1459TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001460 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001461 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001462 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1463 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001464
1465 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001466 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001467 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001468 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001469 fake_encoder_.SetResolutionBitrateLimits(
1470 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1471
1472 VideoEncoderConfig video_encoder_config;
1473 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1474 video_encoder_config.max_bitrate_bps = 0;
1475 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1476 kMaxPayloadLength);
1477
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001478 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001479 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1480 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001481 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1482 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001483 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1484 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1485
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001486 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001487 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1488 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001489 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1490 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001491 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1492 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1493
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001494 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001495 // encoder for 360p should be used.
1496 video_source_.IncomingCapturedFrame(
1497 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1498 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001499 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1500 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001501 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1502 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1503
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001504 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001505 // ignored.
1506 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1507 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001508 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1509 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001510 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1511 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001512 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1513 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001514 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1515 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1516
1517 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1518 // for 270p should be used.
1519 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1520 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001521 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1522 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001523 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1524 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1525
1526 video_stream_encoder_->Stop();
1527}
1528
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001529TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1530 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001531 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1532 DataRate::bps(kTargetBitrateBps), 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001533
1534 VideoEncoderConfig video_encoder_config;
1535 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1536 video_encoder_config.max_bitrate_bps = 0;
1537 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1538 kMaxPayloadLength);
1539
1540 // Encode 720p frame to get the default encoder target bitrate.
1541 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1542 WaitForEncodedFrame(1);
1543 const uint32_t kDefaultTargetBitrateFor720pKbps =
1544 bitrate_allocator_factory_.codec_config()
1545 .simulcastStream[0]
1546 .targetBitrate;
1547
1548 // Set the max recommended encoder bitrate to something lower than the default
1549 // target bitrate.
1550 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1551 1280 * 720, 10 * 1000, 10 * 1000,
1552 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1553 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1554
1555 // Change resolution to trigger encoder reinitialization.
1556 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1557 WaitForEncodedFrame(2);
1558 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1559 WaitForEncodedFrame(3);
1560
1561 // Ensure the target bitrate is capped by the max bitrate.
1562 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1563 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1564 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1565 .simulcastStream[0]
1566 .targetBitrate *
1567 1000,
1568 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1569
1570 video_stream_encoder_->Stop();
1571}
1572
mflodmancc3d4422017-08-03 08:27:51 -07001573TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001574 EXPECT_TRUE(video_source_.has_sinks());
1575 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001576 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001577 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001578 EXPECT_FALSE(video_source_.has_sinks());
1579 EXPECT_TRUE(new_video_source.has_sinks());
1580
mflodmancc3d4422017-08-03 08:27:51 -07001581 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001582}
1583
mflodmancc3d4422017-08-03 08:27:51 -07001584TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001585 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001586 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001587 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001588 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001589}
1590
Jonathan Yubc771b72017-12-08 17:04:29 -08001591TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1592 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001593 const int kWidth = 1280;
1594 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001595
1596 // We rely on the automatic resolution adaptation, but we handle framerate
1597 // adaptation manually by mocking the stats proxy.
1598 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001599
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001600 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001601 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001602 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1603 DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001604 video_stream_encoder_->SetSource(&video_source_,
1605 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001606 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001609 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1610
Jonathan Yubc771b72017-12-08 17:04:29 -08001611 // Adapt down as far as possible.
1612 rtc::VideoSinkWants last_wants;
1613 int64_t t = 1;
1614 int loop_count = 0;
1615 do {
1616 ++loop_count;
1617 last_wants = video_source_.sink_wants();
1618
1619 // Simulate the framerate we've been asked to adapt to.
1620 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1621 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1622 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1623 mock_stats.input_frame_rate = fps;
1624 stats_proxy_->SetMockStats(mock_stats);
1625
1626 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1627 sink_.WaitForEncodedFrame(t);
1628 t += frame_interval_ms;
1629
mflodmancc3d4422017-08-03 08:27:51 -07001630 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001631 VerifyBalancedModeFpsRange(
1632 video_source_.sink_wants(),
1633 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1634 } while (video_source_.sink_wants().max_pixel_count <
1635 last_wants.max_pixel_count ||
1636 video_source_.sink_wants().max_framerate_fps <
1637 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001638
Jonathan Yubc771b72017-12-08 17:04:29 -08001639 // Verify that we've adapted all the way down.
1640 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001641 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001642 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1643 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001644 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001645 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1646 *video_source_.last_sent_height());
1647 EXPECT_EQ(kMinBalancedFramerateFps,
1648 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001649
Jonathan Yubc771b72017-12-08 17:04:29 -08001650 // Adapt back up the same number of times we adapted down.
1651 for (int i = 0; i < loop_count - 1; ++i) {
1652 last_wants = video_source_.sink_wants();
1653
1654 // Simulate the framerate we've been asked to adapt to.
1655 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1656 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1657 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1658 mock_stats.input_frame_rate = fps;
1659 stats_proxy_->SetMockStats(mock_stats);
1660
1661 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1662 sink_.WaitForEncodedFrame(t);
1663 t += frame_interval_ms;
1664
mflodmancc3d4422017-08-03 08:27:51 -07001665 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001666 VerifyBalancedModeFpsRange(
1667 video_source_.sink_wants(),
1668 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1669 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1670 last_wants.max_pixel_count ||
1671 video_source_.sink_wants().max_framerate_fps >
1672 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001673 }
1674
Åsa Persson8c1bf952018-09-13 10:42:19 +02001675 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001676 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001678 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1679 EXPECT_EQ((loop_count - 1) * 2,
1680 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001681
mflodmancc3d4422017-08-03 08:27:51 -07001682 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001683}
mflodmancc3d4422017-08-03 08:27:51 -07001684TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001685 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001686 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1687 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001688 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001689
sprangc5d62e22017-04-02 23:53:04 -07001690 const int kFrameWidth = 1280;
1691 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001692
Åsa Persson8c1bf952018-09-13 10:42:19 +02001693 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001694
kthelgason5e13d412016-12-01 03:59:51 -08001695 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001696 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001697 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001698 frame_timestamp += kFrameIntervalMs;
1699
perkj803d97f2016-11-01 11:45:46 -07001700 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001701 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001702 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001703 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001704 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001705 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001706
asapersson0944a802017-04-07 00:57:58 -07001707 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001708 // wanted resolution.
1709 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1710 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1711 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001712 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001713
1714 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001715 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001716 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001717 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001718
sprangc5d62e22017-04-02 23:53:04 -07001719 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001720 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001721
sprangc5d62e22017-04-02 23:53:04 -07001722 // Force an input frame rate to be available, or the adaptation call won't
1723 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001724 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001725 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001726 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001727 stats_proxy_->SetMockStats(stats);
1728
mflodmancc3d4422017-08-03 08:27:51 -07001729 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001730 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001731 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001732 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001733 frame_timestamp += kFrameIntervalMs;
1734
1735 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001736 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001737 EXPECT_EQ(std::numeric_limits<int>::max(),
1738 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001739 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001740
asapersson02465b82017-04-10 01:12:52 -07001741 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001742 video_stream_encoder_->SetSource(&new_video_source,
1743 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001744 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001745
mflodmancc3d4422017-08-03 08:27:51 -07001746 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001747 new_video_source.IncomingCapturedFrame(
1748 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001749 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001750 frame_timestamp += kFrameIntervalMs;
1751
1752 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001753 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001754
1755 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001756 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001757 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001758 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1759 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001760 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001761 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001762
1763 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001764 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001765 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001766 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1767 EXPECT_EQ(std::numeric_limits<int>::max(),
1768 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001769 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001770
mflodmancc3d4422017-08-03 08:27:51 -07001771 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001772}
1773
mflodmancc3d4422017-08-03 08:27:51 -07001774TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001775 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001776 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1777 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001778
asaperssonfab67072017-04-04 05:51:49 -07001779 const int kWidth = 1280;
1780 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001781 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001782 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001783 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1784 EXPECT_FALSE(stats.bw_limited_resolution);
1785 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1786
1787 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001788 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001789 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001790 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001791
1792 stats = stats_proxy_->GetStats();
1793 EXPECT_TRUE(stats.bw_limited_resolution);
1794 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1795
1796 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001797 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001798 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001799 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001800
1801 stats = stats_proxy_->GetStats();
1802 EXPECT_FALSE(stats.bw_limited_resolution);
1803 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1804 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1805
mflodmancc3d4422017-08-03 08:27:51 -07001806 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001807}
1808
mflodmancc3d4422017-08-03 08:27:51 -07001809TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001810 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001811 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1812 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001813
1814 const int kWidth = 1280;
1815 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001816 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001817 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001818 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1819 EXPECT_FALSE(stats.cpu_limited_resolution);
1820 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1821
1822 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001823 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001824 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001825 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001826
1827 stats = stats_proxy_->GetStats();
1828 EXPECT_TRUE(stats.cpu_limited_resolution);
1829 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1830
1831 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001832 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001833 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001834 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001835
1836 stats = stats_proxy_->GetStats();
1837 EXPECT_FALSE(stats.cpu_limited_resolution);
1838 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001839 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001840
mflodmancc3d4422017-08-03 08:27:51 -07001841 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001842}
1843
mflodmancc3d4422017-08-03 08:27:51 -07001844TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001845 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001846 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1847 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001848
asaperssonfab67072017-04-04 05:51:49 -07001849 const int kWidth = 1280;
1850 const int kHeight = 720;
1851 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001852 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001853 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001854 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001855 EXPECT_FALSE(stats.cpu_limited_resolution);
1856 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1857
asaperssonfab67072017-04-04 05:51:49 -07001858 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001859 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001860 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001861 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001862 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001863 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001864 EXPECT_TRUE(stats.cpu_limited_resolution);
1865 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1866
1867 // Set new source with adaptation still enabled.
1868 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001869 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001870 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001871
asaperssonfab67072017-04-04 05:51:49 -07001872 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001873 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001874 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001875 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001876 EXPECT_TRUE(stats.cpu_limited_resolution);
1877 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1878
1879 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001880 video_stream_encoder_->SetSource(&new_video_source,
1881 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001882
asaperssonfab67072017-04-04 05:51:49 -07001883 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001884 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001885 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001886 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001887 EXPECT_FALSE(stats.cpu_limited_resolution);
1888 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1889
1890 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001891 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001892 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001893
asaperssonfab67072017-04-04 05:51:49 -07001894 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001895 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001896 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001897 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001898 EXPECT_TRUE(stats.cpu_limited_resolution);
1899 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1900
asaperssonfab67072017-04-04 05:51:49 -07001901 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001902 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001903 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001904 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001905 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001906 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001907 EXPECT_FALSE(stats.cpu_limited_resolution);
1908 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001909 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001910
mflodmancc3d4422017-08-03 08:27:51 -07001911 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001912}
1913
mflodmancc3d4422017-08-03 08:27:51 -07001914TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001915 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001916 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1917 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001918
asaperssonfab67072017-04-04 05:51:49 -07001919 const int kWidth = 1280;
1920 const int kHeight = 720;
1921 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001922 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001923 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001924 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001925 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001926 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001927
1928 // Set new source with adaptation still enabled.
1929 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001930 video_stream_encoder_->SetSource(&new_video_source,
1931 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001932
asaperssonfab67072017-04-04 05:51:49 -07001933 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001934 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001935 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001936 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001937 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001938 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001939
asaperssonfab67072017-04-04 05:51:49 -07001940 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001941 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001942 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001943 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001944 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001945 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001946 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001947 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001948
asaperssonfab67072017-04-04 05:51:49 -07001949 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001950 video_stream_encoder_->SetSource(&new_video_source,
1951 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001952
asaperssonfab67072017-04-04 05:51:49 -07001953 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001954 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001955 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001956 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001957 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001958 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001959
asapersson02465b82017-04-10 01:12:52 -07001960 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001961 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001962 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001963
asaperssonfab67072017-04-04 05:51:49 -07001964 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001965 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001966 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001967 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001968 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001969 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1970 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001971
mflodmancc3d4422017-08-03 08:27:51 -07001972 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001973}
1974
mflodmancc3d4422017-08-03 08:27:51 -07001975TEST_F(VideoStreamEncoderTest,
1976 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001977 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001978 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1979 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001980
1981 const int kWidth = 1280;
1982 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001983 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001984 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001985 video_source_.IncomingCapturedFrame(
1986 CreateFrame(timestamp_ms, kWidth, kHeight));
1987 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001988 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1989 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1990 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1991
1992 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001993 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001994 timestamp_ms += kFrameIntervalMs;
1995 video_source_.IncomingCapturedFrame(
1996 CreateFrame(timestamp_ms, kWidth, kHeight));
1997 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001998 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1999 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2000 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2001
2002 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002003 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002004 timestamp_ms += kFrameIntervalMs;
2005 video_source_.IncomingCapturedFrame(
2006 CreateFrame(timestamp_ms, kWidth, kHeight));
2007 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002008 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2009 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2010 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2011
Niels Möller4db138e2018-04-19 09:04:13 +02002012 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002013 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002014
2015 VideoEncoderConfig video_encoder_config;
2016 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2017 // Make format different, to force recreation of encoder.
2018 video_encoder_config.video_format.parameters["foo"] = "foo";
2019 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002020 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002021 timestamp_ms += kFrameIntervalMs;
2022 video_source_.IncomingCapturedFrame(
2023 CreateFrame(timestamp_ms, kWidth, kHeight));
2024 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002025 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2026 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2027 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2028
mflodmancc3d4422017-08-03 08:27:51 -07002029 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002030}
2031
mflodmancc3d4422017-08-03 08:27:51 -07002032TEST_F(VideoStreamEncoderTest,
2033 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002034 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002035 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2036 DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002037
asapersson0944a802017-04-07 00:57:58 -07002038 const int kWidth = 1280;
2039 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002040 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002041
asaperssonfab67072017-04-04 05:51:49 -07002042 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002043 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002044 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002045 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002046 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002047 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2048
asapersson02465b82017-04-10 01:12:52 -07002049 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002050 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002051 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002052 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002053 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002054 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002055 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002056 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2057
2058 // Set new source with adaptation still enabled.
2059 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002060 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002061 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002062
2063 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002064 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002065 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002066 stats = stats_proxy_->GetStats();
2067 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002068 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002069 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2070
sprangc5d62e22017-04-02 23:53:04 -07002071 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002072 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002073 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002074 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002075 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002076 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002077 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002078 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002079 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002080 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002081 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2082
sprangc5d62e22017-04-02 23:53:04 -07002083 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002084 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002085 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2086 mock_stats.input_frame_rate = 30;
2087 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002088 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002089 stats_proxy_->ResetMockStats();
2090
2091 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002092 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002093 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002094
2095 // Framerate now adapted.
2096 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002097 EXPECT_FALSE(stats.cpu_limited_resolution);
2098 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002099 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2100
2101 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002102 video_stream_encoder_->SetSource(&new_video_source,
2103 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002104 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002105 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002106 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002107
2108 stats = stats_proxy_->GetStats();
2109 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002110 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002111 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2112
2113 // Try to trigger overuse. Should not succeed.
2114 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002115 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002116 stats_proxy_->ResetMockStats();
2117
2118 stats = stats_proxy_->GetStats();
2119 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002120 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002121 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2122
2123 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002124 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002125 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002126 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002127 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002128 stats = stats_proxy_->GetStats();
2129 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002130 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002131 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002132
2133 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002134 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002135 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002136 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002137 stats = stats_proxy_->GetStats();
2138 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002139 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002140 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2141
2142 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002143 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002144 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002145 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002146 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002147 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002148 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002149 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002150 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002151 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002152 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2153
2154 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002155 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002156 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002157 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002158 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002159 stats = stats_proxy_->GetStats();
2160 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002161 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002162 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002163 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002164
mflodmancc3d4422017-08-03 08:27:51 -07002165 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002166}
2167
mflodmancc3d4422017-08-03 08:27:51 -07002168TEST_F(VideoStreamEncoderTest,
2169 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002170 const int kWidth = 1280;
2171 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002172 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002173 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2174 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002175
asaperssonfab67072017-04-04 05:51:49 -07002176 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002177 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002178
asaperssonfab67072017-04-04 05:51:49 -07002179 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002180 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002181
asaperssonfab67072017-04-04 05:51:49 -07002182 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002183 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002184
asaperssonfab67072017-04-04 05:51:49 -07002185 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002186 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002187
kthelgason876222f2016-11-29 01:44:11 -08002188 // Expect a scale down.
2189 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002190 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002191
asapersson02465b82017-04-10 01:12:52 -07002192 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002193 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002194 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002195 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002196
asaperssonfab67072017-04-04 05:51:49 -07002197 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002198 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002199 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002200 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002201
asaperssonfab67072017-04-04 05:51:49 -07002202 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002203 EXPECT_EQ(std::numeric_limits<int>::max(),
2204 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002205
asaperssonfab67072017-04-04 05:51:49 -07002206 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002207 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002208 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002209 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002210
asapersson02465b82017-04-10 01:12:52 -07002211 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002212 EXPECT_EQ(std::numeric_limits<int>::max(),
2213 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002214
mflodmancc3d4422017-08-03 08:27:51 -07002215 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002216}
2217
mflodmancc3d4422017-08-03 08:27:51 -07002218TEST_F(VideoStreamEncoderTest,
2219 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002220 const int kWidth = 1280;
2221 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002222 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002223 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2224 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002225
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002226 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002227 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002228 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002229 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002230
2231 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002232 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002233 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002234 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2235 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2236
2237 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002238 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002239 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002240 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2241 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2242 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2243
2244 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002245 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002246 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2247 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2248 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2249
mflodmancc3d4422017-08-03 08:27:51 -07002250 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002251}
2252
mflodmancc3d4422017-08-03 08:27:51 -07002253TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002254 const int kWidth = 1280;
2255 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002256 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002257 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2258 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002259
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002260 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002261 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002262 video_stream_encoder_->SetSource(&source,
2263 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002264 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2265 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002266 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002267
2268 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002269 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002270 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2271 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2272 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2273 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2274
2275 // Trigger adapt down for same input resolution, expect no change.
2276 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2277 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002278 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002279 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2280 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2281 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2282
2283 // Trigger adapt down for larger input resolution, expect no change.
2284 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2285 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002286 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002287 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2288 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2289 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2290
mflodmancc3d4422017-08-03 08:27:51 -07002291 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002292}
2293
mflodmancc3d4422017-08-03 08:27:51 -07002294TEST_F(VideoStreamEncoderTest,
2295 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002296 const int kWidth = 1280;
2297 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002298 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002299 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2300 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002301
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002302 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002303 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002304 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002305 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002306
2307 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002308 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002309 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002310 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2311 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2312
2313 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002314 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002315 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002316 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2317 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2318
mflodmancc3d4422017-08-03 08:27:51 -07002319 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002320}
2321
mflodmancc3d4422017-08-03 08:27:51 -07002322TEST_F(VideoStreamEncoderTest,
2323 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002324 const int kWidth = 1280;
2325 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002326 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002327 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2328 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002329
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002330 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002331 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002332 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002333 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002334
2335 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002336 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002337 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002338 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002339 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2340
2341 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002342 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002343 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002344 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002345 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2346
mflodmancc3d4422017-08-03 08:27:51 -07002347 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002348}
2349
mflodmancc3d4422017-08-03 08:27:51 -07002350TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002351 const int kWidth = 1280;
2352 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002353 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002354 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2355 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002356
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002357 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002358 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002359 video_stream_encoder_->SetSource(&source,
2360 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002361
2362 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2363 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002364 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2366 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2367 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2368
2369 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002370 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002371 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002372 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2373 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2374 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2375
mflodmancc3d4422017-08-03 08:27:51 -07002376 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002377}
2378
mflodmancc3d4422017-08-03 08:27:51 -07002379TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002380 const int kWidth = 1280;
2381 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002382 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002383 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2384 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002385
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002386 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002387 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002388 video_stream_encoder_->SetSource(&source,
2389 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002390
2391 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2392 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002393 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002394 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2395 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2396 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2397
2398 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002399 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002400 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2402 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2403 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2404
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002406}
2407
mflodmancc3d4422017-08-03 08:27:51 -07002408TEST_F(VideoStreamEncoderTest,
2409 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002410 const int kWidth = 1280;
2411 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002412 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002413 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2414 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002415
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002416 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002417 AdaptingFrameForwarder source;
2418 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002419 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002420 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002421
2422 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002423 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002424 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002425 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2426 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2427
2428 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002429 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002430 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002431 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002432 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2434 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2435
2436 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002437 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002438 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002439 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2440 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2441 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2442
mflodmancc3d4422017-08-03 08:27:51 -07002443 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002444}
2445
mflodmancc3d4422017-08-03 08:27:51 -07002446TEST_F(VideoStreamEncoderTest,
2447 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002448 const int kWidth = 1280;
2449 const int kHeight = 720;
2450 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002451 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002452 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2453 DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002454
2455 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2456 stats.input_frame_rate = kInputFps;
2457 stats_proxy_->SetMockStats(stats);
2458
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002459 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002460 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2461 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002462 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002463
2464 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002465 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002466 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2467 sink_.WaitForEncodedFrame(2);
2468 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2469
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002470 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002471 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002472 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002473 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002474 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002475
2476 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002477 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002478 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2479 sink_.WaitForEncodedFrame(3);
2480 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2481
2482 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002483 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002484 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002485
mflodmancc3d4422017-08-03 08:27:51 -07002486 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002487}
2488
mflodmancc3d4422017-08-03 08:27:51 -07002489TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002490 const int kWidth = 1280;
2491 const int kHeight = 720;
2492 const size_t kNumFrames = 10;
2493
Erik Språng4c6ca302019-04-08 15:14:01 +02002494 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002495 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2496 DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002497
asaperssond0de2952017-04-21 01:47:31 -07002498 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002499 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002500 video_source_.set_adaptation_enabled(true);
2501
2502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2503 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2504
2505 int downscales = 0;
2506 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002507 video_source_.IncomingCapturedFrame(
2508 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2509 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002510
asaperssonfab67072017-04-04 05:51:49 -07002511 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002512 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002513 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002514 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002515
2516 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2517 ++downscales;
2518
2519 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2520 EXPECT_EQ(downscales,
2521 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2522 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002523 }
mflodmancc3d4422017-08-03 08:27:51 -07002524 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002525}
2526
mflodmancc3d4422017-08-03 08:27:51 -07002527TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002528 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2529 const int kWidth = 1280;
2530 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002531 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002532 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2533 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002534
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002535 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002536 AdaptingFrameForwarder source;
2537 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002538 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002539 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002540
Åsa Persson8c1bf952018-09-13 10:42:19 +02002541 int64_t timestamp_ms = kFrameIntervalMs;
2542 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002543 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002544 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002545 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2546 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2547
2548 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002549 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002550 timestamp_ms += kFrameIntervalMs;
2551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2552 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002553 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002554 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2555 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2556
2557 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002558 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002559 timestamp_ms += kFrameIntervalMs;
2560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002561 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002562 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002563 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2564 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2565
2566 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002567 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002568 timestamp_ms += kFrameIntervalMs;
2569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2570 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002571 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002572 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2573 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2574
2575 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002576 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002577 timestamp_ms += kFrameIntervalMs;
2578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002579 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002580 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002581 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2582 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2583
mflodmancc3d4422017-08-03 08:27:51 -07002584 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002585}
2586
mflodmancc3d4422017-08-03 08:27:51 -07002587TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002588 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2589 const int kWidth = 1280;
2590 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002591 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002592 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2593 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002594
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002595 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002596 AdaptingFrameForwarder source;
2597 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002598 video_stream_encoder_->SetSource(&source,
2599 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002600
Åsa Persson8c1bf952018-09-13 10:42:19 +02002601 int64_t timestamp_ms = kFrameIntervalMs;
2602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002603 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002604 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002605 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2606 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2607
2608 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002609 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002610 timestamp_ms += kFrameIntervalMs;
2611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2612 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002613 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2615 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2616
2617 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002618 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002619 timestamp_ms += kFrameIntervalMs;
2620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002621 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002622 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2624 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2625
2626 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002627 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002628 timestamp_ms += kFrameIntervalMs;
2629 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2630 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002631 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2633 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2634
2635 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002636 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002637 timestamp_ms += kFrameIntervalMs;
2638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002639 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002640 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2642 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2643
mflodmancc3d4422017-08-03 08:27:51 -07002644 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002645}
2646
Åsa Persson1b247f12019-08-14 17:26:39 +02002647TEST_F(VideoStreamEncoderTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
2648 webrtc::test::ScopedFieldTrials field_trials(
2649 "WebRTC-Video-BalancedDegradationSettings/"
2650 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
2651 // Reset encoder for field trials to take effect.
2652 ConfigureEncoder(video_encoder_config_.Copy());
2653
2654 const int kWidth = 640; // pixels:640x360=230400
2655 const int kHeight = 360;
2656 const int64_t kFrameIntervalMs = 150;
2657 const int kMinBitrateBps = 425000;
2658 const int kTooLowMinBitrateBps = 424000;
Florent Castellia8336d32019-09-09 13:36:55 +02002659 video_stream_encoder_->OnBitrateUpdated(
2660 DataRate::bps(kTooLowMinBitrateBps), DataRate::bps(kTooLowMinBitrateBps),
2661 DataRate::bps(kTooLowMinBitrateBps), 0, 0);
Åsa Persson1b247f12019-08-14 17:26:39 +02002662
2663 // Enable BALANCED preference, no initial limitation.
2664 AdaptingFrameForwarder source;
2665 source.set_adaptation_enabled(true);
2666 video_stream_encoder_->SetSource(&source,
2667 webrtc::DegradationPreference::BALANCED);
2668
2669 int64_t timestamp_ms = kFrameIntervalMs;
2670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2671 sink_.WaitForEncodedFrame(kWidth, kHeight);
2672 VerifyFpsMaxResolutionMax(source.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002673 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2674
2675 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2676 video_stream_encoder_->TriggerQualityLow();
2677 timestamp_ms += kFrameIntervalMs;
2678 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2679 sink_.WaitForEncodedFrame(timestamp_ms);
2680 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02002681 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2682
2683 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2684 video_stream_encoder_->TriggerQualityLow();
2685 timestamp_ms += kFrameIntervalMs;
2686 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2687 sink_.WaitForEncodedFrame(timestamp_ms);
2688 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002689 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2690
Åsa Persson30ab0152019-08-27 12:22:33 +02002691 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2692 video_stream_encoder_->TriggerQualityLow();
2693 timestamp_ms += kFrameIntervalMs;
2694 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2695 sink_.WaitForEncodedFrame(timestamp_ms);
2696 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2697 EXPECT_EQ(source.sink_wants().max_framerate_fps, 10);
2698 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2699
2700 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02002701 video_stream_encoder_->TriggerQualityHigh();
2702 timestamp_ms += kFrameIntervalMs;
2703 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2704 sink_.WaitForEncodedFrame(timestamp_ms);
Åsa Persson30ab0152019-08-27 12:22:33 +02002705 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02002706
Åsa Persson30ab0152019-08-27 12:22:33 +02002707 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02002708 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002709 DataRate::bps(kMinBitrateBps),
Åsa Persson1b247f12019-08-14 17:26:39 +02002710 DataRate::bps(kMinBitrateBps), 0, 0);
2711 video_stream_encoder_->TriggerQualityHigh();
2712 timestamp_ms += kFrameIntervalMs;
2713 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2714 sink_.WaitForEncodedFrame(timestamp_ms);
Åsa Persson30ab0152019-08-27 12:22:33 +02002715 EXPECT_EQ(source.sink_wants().max_framerate_fps, 14);
2716 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2717
2718 video_stream_encoder_->Stop();
2719}
2720
2721TEST_F(VideoStreamEncoderTest,
2722 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
2723 webrtc::test::ScopedFieldTrials field_trials(
2724 "WebRTC-Video-BalancedDegradationSettings/"
2725 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
2726 // Reset encoder for field trials to take effect.
2727 ConfigureEncoder(video_encoder_config_.Copy());
2728
2729 const int kWidth = 640; // pixels:640x360=230400
2730 const int kHeight = 360;
2731 const int64_t kFrameIntervalMs = 150;
2732 const int kResolutionMinBitrateBps = 435000;
2733 const int kTooLowMinResolutionBitrateBps = 434000;
2734 video_stream_encoder_->OnBitrateUpdated(
2735 DataRate::bps(kTooLowMinResolutionBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002736 DataRate::bps(kTooLowMinResolutionBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002737 DataRate::bps(kTooLowMinResolutionBitrateBps), 0, 0);
2738
2739 // Enable BALANCED preference, no initial limitation.
2740 AdaptingFrameForwarder source;
2741 source.set_adaptation_enabled(true);
2742 video_stream_encoder_->SetSource(&source,
2743 webrtc::DegradationPreference::BALANCED);
2744
2745 int64_t timestamp_ms = kFrameIntervalMs;
2746 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2747 sink_.WaitForEncodedFrame(kWidth, kHeight);
2748 VerifyFpsMaxResolutionMax(source.sink_wants());
2749 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2750
2751 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2752 video_stream_encoder_->TriggerQualityLow();
2753 timestamp_ms += kFrameIntervalMs;
2754 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2755 sink_.WaitForEncodedFrame(timestamp_ms);
2756 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2757 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2758
2759 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2760 video_stream_encoder_->TriggerQualityLow();
2761 timestamp_ms += kFrameIntervalMs;
2762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2763 sink_.WaitForEncodedFrame(timestamp_ms);
2764 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2765 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2766
2767 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2768 video_stream_encoder_->TriggerQualityLow();
2769 timestamp_ms += kFrameIntervalMs;
2770 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2771 sink_.WaitForEncodedFrame(timestamp_ms);
2772 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002773 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2774
Åsa Persson30ab0152019-08-27 12:22:33 +02002775 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
2776 video_stream_encoder_->TriggerQualityHigh();
2777 timestamp_ms += kFrameIntervalMs;
2778 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2779 sink_.WaitForEncodedFrame(timestamp_ms);
2780 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2781 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2782
2783 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2784 video_stream_encoder_->TriggerQualityHigh();
2785 timestamp_ms += kFrameIntervalMs;
2786 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2787 sink_.WaitForEncodedFrame(timestamp_ms);
2788 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2789
2790 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
2791 video_stream_encoder_->OnBitrateUpdated(
2792 DataRate::bps(kResolutionMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002793 DataRate::bps(kResolutionMinBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002794 DataRate::bps(kResolutionMinBitrateBps), 0, 0);
2795 video_stream_encoder_->TriggerQualityHigh();
2796 timestamp_ms += kFrameIntervalMs;
2797 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2798 sink_.WaitForEncodedFrame(timestamp_ms);
2799 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2800 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2801
2802 video_stream_encoder_->Stop();
2803}
2804
2805TEST_F(VideoStreamEncoderTest,
2806 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
2807 webrtc::test::ScopedFieldTrials field_trials(
2808 "WebRTC-Video-BalancedDegradationSettings/"
2809 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
2810 // Reset encoder for field trials to take effect.
2811 ConfigureEncoder(video_encoder_config_.Copy());
2812
2813 const int kWidth = 640; // pixels:640x360=230400
2814 const int kHeight = 360;
2815 const int64_t kFrameIntervalMs = 150;
2816 const int kMinBitrateBps = 425000;
2817 const int kTooLowMinBitrateBps = 424000;
2818 const int kResolutionMinBitrateBps = 435000;
2819 const int kTooLowMinResolutionBitrateBps = 434000;
Florent Castellia8336d32019-09-09 13:36:55 +02002820 video_stream_encoder_->OnBitrateUpdated(
2821 DataRate::bps(kTooLowMinBitrateBps), DataRate::bps(kTooLowMinBitrateBps),
2822 DataRate::bps(kTooLowMinBitrateBps), 0, 0);
Åsa Persson30ab0152019-08-27 12:22:33 +02002823
2824 // Enable BALANCED preference, no initial limitation.
2825 AdaptingFrameForwarder source;
2826 source.set_adaptation_enabled(true);
2827 video_stream_encoder_->SetSource(&source,
2828 webrtc::DegradationPreference::BALANCED);
2829
2830 int64_t timestamp_ms = kFrameIntervalMs;
2831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2832 sink_.WaitForEncodedFrame(kWidth, kHeight);
2833 VerifyFpsMaxResolutionMax(source.sink_wants());
2834 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2835
2836 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2837 video_stream_encoder_->TriggerQualityLow();
2838 timestamp_ms += kFrameIntervalMs;
2839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2840 sink_.WaitForEncodedFrame(timestamp_ms);
2841 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2842 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2843
2844 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2845 video_stream_encoder_->TriggerQualityLow();
2846 timestamp_ms += kFrameIntervalMs;
2847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2848 sink_.WaitForEncodedFrame(timestamp_ms);
2849 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2850 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2851
2852 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2853 video_stream_encoder_->TriggerQualityLow();
2854 timestamp_ms += kFrameIntervalMs;
2855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2856 sink_.WaitForEncodedFrame(timestamp_ms);
2857 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2858 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2859
2860 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
2861 video_stream_encoder_->TriggerQualityHigh();
2862 timestamp_ms += kFrameIntervalMs;
2863 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2864 sink_.WaitForEncodedFrame(timestamp_ms);
2865 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2866
2867 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
2868 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002869 DataRate::bps(kMinBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002870 DataRate::bps(kMinBitrateBps), 0, 0);
2871 video_stream_encoder_->TriggerQualityHigh();
2872 timestamp_ms += kFrameIntervalMs;
2873 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2874 sink_.WaitForEncodedFrame(timestamp_ms);
2875 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2876 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2877
2878 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2879 video_stream_encoder_->OnBitrateUpdated(
2880 DataRate::bps(kTooLowMinResolutionBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002881 DataRate::bps(kTooLowMinResolutionBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002882 DataRate::bps(kTooLowMinResolutionBitrateBps), 0, 0);
2883 video_stream_encoder_->TriggerQualityHigh();
2884 timestamp_ms += kFrameIntervalMs;
2885 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2886 sink_.WaitForEncodedFrame(timestamp_ms);
2887 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2888
2889 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
2890 video_stream_encoder_->OnBitrateUpdated(
2891 DataRate::bps(kResolutionMinBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02002892 DataRate::bps(kResolutionMinBitrateBps),
Åsa Persson30ab0152019-08-27 12:22:33 +02002893 DataRate::bps(kResolutionMinBitrateBps), 0, 0);
2894 video_stream_encoder_->TriggerQualityHigh();
2895 timestamp_ms += kFrameIntervalMs;
2896 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2897 sink_.WaitForEncodedFrame(timestamp_ms);
2898 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2899 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2900
Åsa Persson1b247f12019-08-14 17:26:39 +02002901 video_stream_encoder_->Stop();
2902}
2903
mflodmancc3d4422017-08-03 08:27:51 -07002904TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002905 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2906 const int kWidth = 1280;
2907 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002908 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002909 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
2910 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002911
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002912 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002913 AdaptingFrameForwarder source;
2914 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002915 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002916 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002917
Åsa Persson8c1bf952018-09-13 10:42:19 +02002918 int64_t timestamp_ms = kFrameIntervalMs;
2919 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002920 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002921 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2923 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2924 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2925 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2926
2927 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002929 timestamp_ms += kFrameIntervalMs;
2930 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2931 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002932 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002933 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2934 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2935 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2936 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2937
2938 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002939 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002940 timestamp_ms += kFrameIntervalMs;
2941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2942 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002943 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002944 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2945 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2946 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2947 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2948
Jonathan Yubc771b72017-12-08 17:04:29 -08002949 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002951 timestamp_ms += kFrameIntervalMs;
2952 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2953 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002954 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002955 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2956 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002957 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002958 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2959
Jonathan Yubc771b72017-12-08 17:04:29 -08002960 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002961 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002962 timestamp_ms += kFrameIntervalMs;
2963 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2964 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002965 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002966 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002967 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2968 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2969 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2970 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2971
Jonathan Yubc771b72017-12-08 17:04:29 -08002972 // Trigger quality adapt down, expect no change (min resolution reached).
2973 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002974 timestamp_ms += kFrameIntervalMs;
2975 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2976 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002977 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2978 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2979 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2980 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2981 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2982
2983 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002985 timestamp_ms += kFrameIntervalMs;
2986 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2987 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002988 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002989 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2990 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2991 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2992 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2993
2994 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2995 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002996 timestamp_ms += kFrameIntervalMs;
2997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2998 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002999 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3000 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3001 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3002 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3003 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3004
3005 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3006 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003007 timestamp_ms += kFrameIntervalMs;
3008 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3009 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003010 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003011 last_wants = source.sink_wants();
3012 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3013 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003014 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003015 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3016
3017 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003018 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003019 timestamp_ms += kFrameIntervalMs;
3020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3021 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003022 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003023 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3024 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003025 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003026 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3027
3028 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003029 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003030 timestamp_ms += kFrameIntervalMs;
3031 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003032 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003033 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003034 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003035 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3036 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003037 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003038 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003039
mflodmancc3d4422017-08-03 08:27:51 -07003040 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003041}
3042
mflodmancc3d4422017-08-03 08:27:51 -07003043TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003044 const int kWidth = 640;
3045 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003046
Erik Språng4c6ca302019-04-08 15:14:01 +02003047 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003048 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3049 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003050
perkj803d97f2016-11-01 11:45:46 -07003051 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003052 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003053 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003054 }
3055
mflodmancc3d4422017-08-03 08:27:51 -07003056 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003057 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003058 video_source_.IncomingCapturedFrame(CreateFrame(
3059 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003060 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003061 }
3062
mflodmancc3d4422017-08-03 08:27:51 -07003063 video_stream_encoder_->Stop();
3064 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003065 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003066
perkj803d97f2016-11-01 11:45:46 -07003067 EXPECT_EQ(1,
3068 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3069 EXPECT_EQ(
3070 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3071}
3072
mflodmancc3d4422017-08-03 08:27:51 -07003073TEST_F(VideoStreamEncoderTest,
3074 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003075 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003076 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3077 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003078 const int kWidth = 640;
3079 const int kHeight = 360;
3080
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003081 video_stream_encoder_->SetSource(&video_source_,
3082 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003083
3084 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3085 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003086 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003087 }
3088
mflodmancc3d4422017-08-03 08:27:51 -07003089 video_stream_encoder_->Stop();
3090 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003091 stats_proxy_.reset();
3092
3093 EXPECT_EQ(0,
3094 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3095}
3096
mflodmancc3d4422017-08-03 08:27:51 -07003097TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003098 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003099 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003100
3101 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003102 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08003103 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003104 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3105 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003106
sprang57c2fff2017-01-16 06:24:02 -08003107 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003108 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003109 video_stream_encoder_->OnBitrateUpdated(
3110 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3111 DataRate::bps(kLowTargetBitrateBps), 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003112
sprang57c2fff2017-01-16 06:24:02 -08003113 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003114 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3115 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003116 VideoBitrateAllocation bitrate_allocation =
3117 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003118 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003119 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003120 // TODO(srte): The use of millisecs here looks like an error, but the tests
3121 // fails using seconds, this should be investigated.
3122 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003123
3124 // Not called on second frame.
3125 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3126 .Times(0);
3127 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003128 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3129 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02003130 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003131
3132 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003133 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3134 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003135 const int64_t start_time_ms = rtc::TimeMillis();
3136 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3137 video_source_.IncomingCapturedFrame(
3138 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3139 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02003140 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003141 }
3142
3143 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003144 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003145
mflodmancc3d4422017-08-03 08:27:51 -07003146 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003147}
3148
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003149TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3150 // 2 TLs configured, temporal layers supported by encoder.
3151 const int kNumTemporalLayers = 2;
3152 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3153 fake_encoder_.SetTemporalLayersSupported(0, true);
3154
3155 // Bitrate allocated across temporal layers.
3156 const int kTl0Bps = kTargetBitrateBps *
3157 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3158 kNumTemporalLayers, /*temporal_id*/ 0);
3159 const int kTl1Bps = kTargetBitrateBps *
3160 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3161 kNumTemporalLayers, /*temporal_id*/ 1);
3162 VideoBitrateAllocation expected_bitrate;
3163 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3164 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3165
3166 VerifyAllocatedBitrate(expected_bitrate);
3167 video_stream_encoder_->Stop();
3168}
3169
3170TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3171 // 2 TLs configured, temporal layers not supported by encoder.
3172 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3173 fake_encoder_.SetTemporalLayersSupported(0, false);
3174
3175 // Temporal layers not supported by the encoder.
3176 // Total bitrate should be at ti:0.
3177 VideoBitrateAllocation expected_bitrate;
3178 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3179
3180 VerifyAllocatedBitrate(expected_bitrate);
3181 video_stream_encoder_->Stop();
3182}
3183
3184TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3185 // 2 TLs configured, temporal layers only supported for first stream.
3186 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3187 fake_encoder_.SetTemporalLayersSupported(0, true);
3188 fake_encoder_.SetTemporalLayersSupported(1, false);
3189
3190 const int kS0Bps = 150000;
3191 const int kS0Tl0Bps =
3192 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3193 /*num_layers*/ 2, /*temporal_id*/ 0);
3194 const int kS0Tl1Bps =
3195 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3196 /*num_layers*/ 2, /*temporal_id*/ 1);
3197 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3198 // Temporal layers not supported by si:1.
3199 VideoBitrateAllocation expected_bitrate;
3200 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3201 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3202 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3203
3204 VerifyAllocatedBitrate(expected_bitrate);
3205 video_stream_encoder_->Stop();
3206}
3207
Niels Möller7dc26b72017-12-06 10:27:48 +01003208TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3209 const int kFrameWidth = 1280;
3210 const int kFrameHeight = 720;
3211 const int kFramerate = 24;
3212
Erik Språng4c6ca302019-04-08 15:14:01 +02003213 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003214 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3215 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003216 test::FrameForwarder source;
3217 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003218 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003219
3220 // Insert a single frame, triggering initial configuration.
3221 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3222 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3223
3224 EXPECT_EQ(
3225 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3226 kDefaultFramerate);
3227
3228 // Trigger reconfigure encoder (without resetting the entire instance).
3229 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003230 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003231 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3232 video_encoder_config.number_of_streams = 1;
3233 video_encoder_config.video_stream_factory =
3234 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3235 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003236 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003237 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3238
3239 // Detector should be updated with fps limit from codec config.
3240 EXPECT_EQ(
3241 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3242 kFramerate);
3243
3244 // Trigger overuse, max framerate should be reduced.
3245 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3246 stats.input_frame_rate = kFramerate;
3247 stats_proxy_->SetMockStats(stats);
3248 video_stream_encoder_->TriggerCpuOveruse();
3249 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3250 int adapted_framerate =
3251 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3252 EXPECT_LT(adapted_framerate, kFramerate);
3253
3254 // Trigger underuse, max framerate should go back to codec configured fps.
3255 // Set extra low fps, to make sure it's actually reset, not just incremented.
3256 stats = stats_proxy_->GetStats();
3257 stats.input_frame_rate = adapted_framerate / 2;
3258 stats_proxy_->SetMockStats(stats);
3259 video_stream_encoder_->TriggerCpuNormalUsage();
3260 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3261 EXPECT_EQ(
3262 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3263 kFramerate);
3264
3265 video_stream_encoder_->Stop();
3266}
3267
3268TEST_F(VideoStreamEncoderTest,
3269 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3270 const int kFrameWidth = 1280;
3271 const int kFrameHeight = 720;
3272 const int kLowFramerate = 15;
3273 const int kHighFramerate = 25;
3274
Erik Språng4c6ca302019-04-08 15:14:01 +02003275 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003276 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3277 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003278 test::FrameForwarder source;
3279 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003280 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003281
3282 // Trigger initial configuration.
3283 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003284 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003285 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3286 video_encoder_config.number_of_streams = 1;
3287 video_encoder_config.video_stream_factory =
3288 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3289 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3290 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003291 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003292 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3293
3294 EXPECT_EQ(
3295 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3296 kLowFramerate);
3297
3298 // Trigger overuse, max framerate should be reduced.
3299 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3300 stats.input_frame_rate = kLowFramerate;
3301 stats_proxy_->SetMockStats(stats);
3302 video_stream_encoder_->TriggerCpuOveruse();
3303 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3304 int adapted_framerate =
3305 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3306 EXPECT_LT(adapted_framerate, kLowFramerate);
3307
3308 // Reconfigure the encoder with a new (higher max framerate), max fps should
3309 // still respect the adaptation.
3310 video_encoder_config.video_stream_factory =
3311 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3312 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3313 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003314 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003315 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3316
3317 EXPECT_EQ(
3318 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3319 adapted_framerate);
3320
3321 // Trigger underuse, max framerate should go back to codec configured fps.
3322 stats = stats_proxy_->GetStats();
3323 stats.input_frame_rate = adapted_framerate;
3324 stats_proxy_->SetMockStats(stats);
3325 video_stream_encoder_->TriggerCpuNormalUsage();
3326 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3327 EXPECT_EQ(
3328 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3329 kHighFramerate);
3330
3331 video_stream_encoder_->Stop();
3332}
3333
mflodmancc3d4422017-08-03 08:27:51 -07003334TEST_F(VideoStreamEncoderTest,
3335 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003336 const int kFrameWidth = 1280;
3337 const int kFrameHeight = 720;
3338 const int kFramerate = 24;
3339
Erik Språng4c6ca302019-04-08 15:14:01 +02003340 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003341 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3342 DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003343 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003344 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003345 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003346
3347 // Trigger initial configuration.
3348 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003349 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003350 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3351 video_encoder_config.number_of_streams = 1;
3352 video_encoder_config.video_stream_factory =
3353 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3354 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003355 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003356 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003357 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003358
Niels Möller7dc26b72017-12-06 10:27:48 +01003359 EXPECT_EQ(
3360 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3361 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003362
3363 // Trigger overuse, max framerate should be reduced.
3364 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3365 stats.input_frame_rate = kFramerate;
3366 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003367 video_stream_encoder_->TriggerCpuOveruse();
3368 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003369 int adapted_framerate =
3370 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003371 EXPECT_LT(adapted_framerate, kFramerate);
3372
3373 // Change degradation preference to not enable framerate scaling. Target
3374 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003375 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003376 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003377 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003378 EXPECT_EQ(
3379 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3380 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003381
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003383}
3384
mflodmancc3d4422017-08-03 08:27:51 -07003385TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003386 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003387 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003388 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003389 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003390 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003391 const int kWidth = 640;
3392 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003393
asaperssonfab67072017-04-04 05:51:49 -07003394 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003395
3396 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003397 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003398
3399 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003400 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003401
sprangc5d62e22017-04-02 23:53:04 -07003402 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003403
asaperssonfab67072017-04-04 05:51:49 -07003404 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003405 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003406 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003407
3408 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003409 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003410
sprangc5d62e22017-04-02 23:53:04 -07003411 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003412
mflodmancc3d4422017-08-03 08:27:51 -07003413 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003414}
3415
mflodmancc3d4422017-08-03 08:27:51 -07003416TEST_F(VideoStreamEncoderTest,
3417 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003418 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003419 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003420 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003421 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003422 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003423 const int kWidth = 640;
3424 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003425
3426 // We expect the n initial frames to get dropped.
3427 int i;
3428 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003429 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003430 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003431 }
3432 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003433 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003434 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003435
3436 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003437 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003438
mflodmancc3d4422017-08-03 08:27:51 -07003439 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003440}
3441
mflodmancc3d4422017-08-03 08:27:51 -07003442TEST_F(VideoStreamEncoderTest,
3443 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003444 const int kWidth = 640;
3445 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003446 video_stream_encoder_->OnBitrateUpdated(
3447 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3448 DataRate::bps(kLowTargetBitrateBps), 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003449
3450 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003451 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003452 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003453
asaperssonfab67072017-04-04 05:51:49 -07003454 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003455 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003456 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003457
mflodmancc3d4422017-08-03 08:27:51 -07003458 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003459}
3460
mflodmancc3d4422017-08-03 08:27:51 -07003461TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003462 const int kWidth = 640;
3463 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003464 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003465
3466 VideoEncoderConfig video_encoder_config;
3467 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3468 // Make format different, to force recreation of encoder.
3469 video_encoder_config.video_format.parameters["foo"] = "foo";
3470 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003471 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003472 video_stream_encoder_->OnBitrateUpdated(
3473 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
3474 DataRate::bps(kLowTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003475
kthelgasonb83797b2017-02-14 11:57:25 -08003476 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003477 video_stream_encoder_->SetSource(&video_source_,
3478 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003479
asaperssonfab67072017-04-04 05:51:49 -07003480 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003481 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003482 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003483
mflodmancc3d4422017-08-03 08:27:51 -07003484 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003485 fake_encoder_.SetQualityScaling(true);
3486}
3487
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003488TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
3489 webrtc::test::ScopedFieldTrials field_trials(
3490 "WebRTC-InitialFramedrop/Enabled/");
3491 // Reset encoder for field trials to take effect.
3492 ConfigureEncoder(video_encoder_config_.Copy());
3493 const int kTooLowBitrateForFrameSizeBps = 10000;
3494 const int kWidth = 640;
3495 const int kHeight = 360;
3496
Erik Språng4c6ca302019-04-08 15:14:01 +02003497 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003498 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3499 DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003500 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3501 // Frame should not be dropped.
3502 WaitForEncodedFrame(1);
3503
Erik Språng610c7632019-03-06 15:37:33 +01003504 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003505 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003506 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003507 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003508 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3509 // Expect to drop this frame, the wait should time out.
3510 ExpectDroppedFrame();
3511
3512 // Expect the sink_wants to specify a scaled frame.
3513 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3514 video_stream_encoder_->Stop();
3515}
3516
Åsa Persson139f4dc2019-08-02 09:29:58 +02003517TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3518 webrtc::test::ScopedFieldTrials field_trials(
3519 "WebRTC-Video-QualityScalerSettings/"
3520 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3521 // Reset encoder for field trials to take effect.
3522 ConfigureEncoder(video_encoder_config_.Copy());
3523 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3524 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3525 const int kWidth = 640;
3526 const int kHeight = 360;
3527
3528 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003529 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3530 DataRate::bps(kTargetBitrateBps), 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003531 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3532 // Frame should not be dropped.
3533 WaitForEncodedFrame(1);
3534
3535 video_stream_encoder_->OnBitrateUpdated(
3536 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003537 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Åsa Persson139f4dc2019-08-02 09:29:58 +02003538 DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0);
3539 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3540 // Frame should not be dropped.
3541 WaitForEncodedFrame(2);
3542
3543 video_stream_encoder_->OnBitrateUpdated(
3544 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003545 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Åsa Persson139f4dc2019-08-02 09:29:58 +02003546 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
3547 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3548 // Expect to drop this frame, the wait should time out.
3549 ExpectDroppedFrame();
3550
3551 // Expect the sink_wants to specify a scaled frame.
3552 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3553 video_stream_encoder_->Stop();
3554}
3555
mflodmancc3d4422017-08-03 08:27:51 -07003556TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003557 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3558 const int kTooSmallWidth = 10;
3559 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02003560 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003561 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3562 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003563
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003564 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003565 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003566 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003567 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003568 VerifyNoLimitation(source.sink_wants());
3569 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3570
3571 // Trigger adapt down, too small frame, expect no change.
3572 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003573 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003574 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003575 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003576 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3577 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3578
mflodmancc3d4422017-08-03 08:27:51 -07003579 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003580}
3581
mflodmancc3d4422017-08-03 08:27:51 -07003582TEST_F(VideoStreamEncoderTest,
3583 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003584 const int kTooSmallWidth = 10;
3585 const int kTooSmallHeight = 10;
3586 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02003587 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003588 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3589 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003590
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003591 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003592 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003593 video_stream_encoder_->SetSource(&source,
3594 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003595 VerifyNoLimitation(source.sink_wants());
3596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3598
3599 // Trigger adapt down, expect limited framerate.
3600 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003601 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003602 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003603 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3604 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3606 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3607
3608 // Trigger adapt down, too small frame, expect no change.
3609 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003610 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003611 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003612 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3613 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3615 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3616
mflodmancc3d4422017-08-03 08:27:51 -07003617 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003618}
3619
mflodmancc3d4422017-08-03 08:27:51 -07003620TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07003621 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02003622 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003623 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3624 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02003625 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07003626 const int kFrameWidth = 1280;
3627 const int kFrameHeight = 720;
3628 video_source_.IncomingCapturedFrame(
3629 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003630 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07003631 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003632}
3633
sprangb1ca0732017-02-01 08:38:12 -08003634// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07003635TEST_F(VideoStreamEncoderTest,
3636 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003637 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003638 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3639 DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08003640
3641 const int kFrameWidth = 1280;
3642 const int kFrameHeight = 720;
3643 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07003644 // requested by
3645 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08003646 video_source_.set_adaptation_enabled(true);
3647
3648 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003649 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003650 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003651
3652 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003653 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08003654 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003655 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003656 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08003657
asaperssonfab67072017-04-04 05:51:49 -07003658 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003659 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08003660 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003661 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003662 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003663
mflodmancc3d4422017-08-03 08:27:51 -07003664 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08003665}
sprangfe627f32017-03-29 08:24:59 -07003666
mflodmancc3d4422017-08-03 08:27:51 -07003667TEST_F(VideoStreamEncoderTest,
3668 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07003669 const int kFrameWidth = 1280;
3670 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07003671
Erik Språng4c6ca302019-04-08 15:14:01 +02003672 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003673 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3674 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003675 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003676 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003677 video_source_.set_adaptation_enabled(true);
3678
sprang4847ae62017-06-27 07:06:52 -07003679 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003680
3681 video_source_.IncomingCapturedFrame(
3682 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003683 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003684
3685 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07003686 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003687
3688 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07003689 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003690 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003691 video_source_.IncomingCapturedFrame(
3692 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003693 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003694 }
3695
3696 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07003697 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003698 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003699 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003700 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003701 video_source_.IncomingCapturedFrame(
3702 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003703 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003704 ++num_frames_dropped;
3705 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003706 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003707 }
3708 }
3709
sprang4847ae62017-06-27 07:06:52 -07003710 // Add some slack to account for frames dropped by the frame dropper.
3711 const int kErrorMargin = 1;
3712 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003713 kErrorMargin);
3714
3715 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07003716 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003717 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003718 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003719 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003720 video_source_.IncomingCapturedFrame(
3721 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003722 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003723 ++num_frames_dropped;
3724 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003725 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003726 }
3727 }
sprang4847ae62017-06-27 07:06:52 -07003728 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07003729 kErrorMargin);
3730
3731 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07003732 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003733 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003734 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003735 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003736 video_source_.IncomingCapturedFrame(
3737 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003738 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003739 ++num_frames_dropped;
3740 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003741 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003742 }
3743 }
sprang4847ae62017-06-27 07:06:52 -07003744 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003745 kErrorMargin);
3746
3747 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07003748 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003749 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003750 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003751 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003752 video_source_.IncomingCapturedFrame(
3753 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003754 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003755 ++num_frames_dropped;
3756 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003757 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003758 }
3759 }
3760 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003763}
3764
mflodmancc3d4422017-08-03 08:27:51 -07003765TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07003766 const int kFramerateFps = 5;
3767 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07003768 const int kFrameWidth = 1280;
3769 const int kFrameHeight = 720;
3770
sprang4847ae62017-06-27 07:06:52 -07003771 // Reconfigure encoder with two temporal layers and screensharing, which will
3772 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02003773 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07003774
Erik Språng4c6ca302019-04-08 15:14:01 +02003775 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003776 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3777 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003778 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003779 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003780 video_source_.set_adaptation_enabled(true);
3781
sprang4847ae62017-06-27 07:06:52 -07003782 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003783
3784 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08003785 rtc::VideoSinkWants last_wants;
3786 do {
3787 last_wants = video_source_.sink_wants();
3788
sprangc5d62e22017-04-02 23:53:04 -07003789 // Insert frames to get a new fps estimate...
3790 for (int j = 0; j < kFramerateFps; ++j) {
3791 video_source_.IncomingCapturedFrame(
3792 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08003793 if (video_source_.last_sent_width()) {
3794 sink_.WaitForEncodedFrame(timestamp_ms);
3795 }
sprangc5d62e22017-04-02 23:53:04 -07003796 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003797 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07003798 }
3799 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07003800 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003801 } while (video_source_.sink_wants().max_framerate_fps <
3802 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003803
Jonathan Yubc771b72017-12-08 17:04:29 -08003804 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003805
mflodmancc3d4422017-08-03 08:27:51 -07003806 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003807}
asaperssonf7e294d2017-06-13 23:25:22 -07003808
mflodmancc3d4422017-08-03 08:27:51 -07003809TEST_F(VideoStreamEncoderTest,
3810 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003811 const int kWidth = 1280;
3812 const int kHeight = 720;
3813 const int64_t kFrameIntervalMs = 150;
3814 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003815 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003816 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3817 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003818
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003819 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003820 AdaptingFrameForwarder source;
3821 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003822 video_stream_encoder_->SetSource(&source,
3823 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003824 timestamp_ms += kFrameIntervalMs;
3825 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003826 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003827 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3831
3832 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003833 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003834 timestamp_ms += kFrameIntervalMs;
3835 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003836 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003837 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3838 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3839 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3840 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3841
3842 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003843 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003844 timestamp_ms += kFrameIntervalMs;
3845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003846 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003847 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3848 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3850 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3851
3852 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003853 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003854 timestamp_ms += kFrameIntervalMs;
3855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003856 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003857 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3858 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3859 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3860 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3861
3862 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003863 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003864 timestamp_ms += kFrameIntervalMs;
3865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003866 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003867 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3868 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3870 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3871
3872 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003873 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003874 timestamp_ms += kFrameIntervalMs;
3875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003876 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003877 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3880 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3881
3882 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003883 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003884 timestamp_ms += kFrameIntervalMs;
3885 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003886 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003887 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3888 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3889 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3890 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3891
3892 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003893 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003894 timestamp_ms += kFrameIntervalMs;
3895 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003896 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003897 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3898 rtc::VideoSinkWants last_wants = source.sink_wants();
3899 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3900 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3901 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3902
3903 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003904 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003905 timestamp_ms += kFrameIntervalMs;
3906 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003907 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003908 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3909 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3910 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3911 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3912
3913 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003914 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003915 timestamp_ms += kFrameIntervalMs;
3916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003917 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003918 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3919 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3921 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3922
3923 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003924 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003925 timestamp_ms += kFrameIntervalMs;
3926 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003927 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003928 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3929 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3930 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3931 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3932
3933 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003934 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003935 timestamp_ms += kFrameIntervalMs;
3936 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003937 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003938 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3939 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3940 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3941 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3942
3943 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003944 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003945 timestamp_ms += kFrameIntervalMs;
3946 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003947 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003948 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3949 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3950 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3951 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3952
3953 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003954 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003955 timestamp_ms += kFrameIntervalMs;
3956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003957 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003958 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3959 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3960 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3961 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3962
3963 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003964 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003965 timestamp_ms += kFrameIntervalMs;
3966 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003967 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003968 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3969 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3970 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3971 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3972
Åsa Persson30ab0152019-08-27 12:22:33 +02003973 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003974 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003975 timestamp_ms += kFrameIntervalMs;
3976 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003977 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003978 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003979 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003980 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3981 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3982 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3983
3984 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003985 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003986 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003987 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3988
mflodmancc3d4422017-08-03 08:27:51 -07003989 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003990}
3991
mflodmancc3d4422017-08-03 08:27:51 -07003992TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003993 const int kWidth = 1280;
3994 const int kHeight = 720;
3995 const int64_t kFrameIntervalMs = 150;
3996 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003997 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003998 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
3999 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004000
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004001 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004002 AdaptingFrameForwarder source;
4003 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004004 video_stream_encoder_->SetSource(&source,
4005 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004006 timestamp_ms += kFrameIntervalMs;
4007 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004008 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004009 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004010 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4011 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4012 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4013 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4014 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4015 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4016
4017 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004018 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004019 timestamp_ms += kFrameIntervalMs;
4020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004021 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004022 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4023 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4025 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4026 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4027 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4029
4030 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004031 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004032 timestamp_ms += kFrameIntervalMs;
4033 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004034 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004035 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4036 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4037 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4038 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4039 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4040 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4041 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4042
4043 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004044 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004045 timestamp_ms += kFrameIntervalMs;
4046 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004047 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004048 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4049 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4050 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4051 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4052 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4053 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4054 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4055
4056 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004057 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004058 timestamp_ms += kFrameIntervalMs;
4059 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004060 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004061 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4062 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4063 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4064 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4065 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4066 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4067 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4068
4069 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004070 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004071 timestamp_ms += kFrameIntervalMs;
4072 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004073 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004074 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4075 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4076 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4077 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4078 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4079 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4080 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4081
4082 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004083 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004084 timestamp_ms += kFrameIntervalMs;
4085 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004086 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004087 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004088 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004089 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4090 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4091 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4092 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4093 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4094 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4095
4096 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004097 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004098 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004099 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4100 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4101
mflodmancc3d4422017-08-03 08:27:51 -07004102 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004103}
4104
mflodmancc3d4422017-08-03 08:27:51 -07004105TEST_F(VideoStreamEncoderTest,
4106 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004107 const int kWidth = 640;
4108 const int kHeight = 360;
4109 const int kFpsLimit = 15;
4110 const int64_t kFrameIntervalMs = 150;
4111 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004112 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004113 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4114 DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004115
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004116 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004117 AdaptingFrameForwarder source;
4118 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004119 video_stream_encoder_->SetSource(&source,
4120 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004121 timestamp_ms += kFrameIntervalMs;
4122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004123 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004124 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004125 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4126 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4127 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4128 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4129 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4130 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4131
4132 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004133 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004134 timestamp_ms += kFrameIntervalMs;
4135 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004136 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004137 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4138 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4139 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4140 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4141 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4142 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4143 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4144
4145 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004146 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004147 timestamp_ms += kFrameIntervalMs;
4148 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004149 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004150 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4151 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4152 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4153 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4154 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4155 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4156 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4157
4158 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004159 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004160 timestamp_ms += kFrameIntervalMs;
4161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004162 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004163 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4164 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4165 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4166 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4167 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4168 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4169 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4170
4171 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004172 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004173 timestamp_ms += kFrameIntervalMs;
4174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004175 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004176 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004177 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4178 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4179 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4180 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4181 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4182 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4183
4184 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004185 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004186 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004187 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4188 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4189
mflodmancc3d4422017-08-03 08:27:51 -07004190 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004191}
4192
mflodmancc3d4422017-08-03 08:27:51 -07004193TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004194 const int kFrameWidth = 1920;
4195 const int kFrameHeight = 1080;
4196 // 3/4 of 1920.
4197 const int kAdaptedFrameWidth = 1440;
4198 // 3/4 of 1080 rounded down to multiple of 4.
4199 const int kAdaptedFrameHeight = 808;
4200 const int kFramerate = 24;
4201
Erik Språng4c6ca302019-04-08 15:14:01 +02004202 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004203 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4204 DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004205 // Trigger reconfigure encoder (without resetting the entire instance).
4206 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004207 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004208 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4209 video_encoder_config.number_of_streams = 1;
4210 video_encoder_config.video_stream_factory =
4211 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004212 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004213 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004214 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004215
4216 video_source_.set_adaptation_enabled(true);
4217
4218 video_source_.IncomingCapturedFrame(
4219 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004220 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004221
4222 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004223 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004224 video_source_.IncomingCapturedFrame(
4225 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004226 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004227
mflodmancc3d4422017-08-03 08:27:51 -07004228 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004229}
4230
mflodmancc3d4422017-08-03 08:27:51 -07004231TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004232 const int kFrameWidth = 1280;
4233 const int kFrameHeight = 720;
4234 const int kLowFps = 2;
4235 const int kHighFps = 30;
4236
Erik Språng4c6ca302019-04-08 15:14:01 +02004237 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004238 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4239 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004240
4241 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4242 max_framerate_ = kLowFps;
4243
4244 // Insert 2 seconds of 2fps video.
4245 for (int i = 0; i < kLowFps * 2; ++i) {
4246 video_source_.IncomingCapturedFrame(
4247 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4248 WaitForEncodedFrame(timestamp_ms);
4249 timestamp_ms += 1000 / kLowFps;
4250 }
4251
4252 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004253 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004254 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4255 DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004256 video_source_.IncomingCapturedFrame(
4257 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4258 WaitForEncodedFrame(timestamp_ms);
4259 timestamp_ms += 1000 / kLowFps;
4260
4261 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4262
4263 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004264 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004265 const int kFrameIntervalMs = 1000 / kHighFps;
4266 max_framerate_ = kHighFps;
4267 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4268 video_source_.IncomingCapturedFrame(
4269 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4270 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4271 // be dropped if the encoder hans't been updated with the new higher target
4272 // framerate yet, causing it to overshoot the target bitrate and then
4273 // suffering the wrath of the media optimizer.
4274 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4275 timestamp_ms += kFrameIntervalMs;
4276 }
4277
4278 // Don expect correct measurement just yet, but it should be higher than
4279 // before.
4280 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4281
mflodmancc3d4422017-08-03 08:27:51 -07004282 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004283}
4284
mflodmancc3d4422017-08-03 08:27:51 -07004285TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004286 const int kFrameWidth = 1280;
4287 const int kFrameHeight = 720;
4288 const int kTargetBitrateBps = 1000000;
4289
4290 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004291 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004292 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004293 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4294 DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004295 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004296
4297 // Insert a first video frame, causes another bitrate update.
4298 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4299 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4300 video_source_.IncomingCapturedFrame(
4301 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4302 WaitForEncodedFrame(timestamp_ms);
4303
4304 // Next, simulate video suspension due to pacer queue overrun.
Florent Castellia8336d32019-09-09 13:36:55 +02004305 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
4306 DataRate::bps(0), 0, 1);
sprang4847ae62017-06-27 07:06:52 -07004307
4308 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004309 timestamp_ms += kProcessIntervalMs;
4310 fake_clock_.AdvanceTime(TimeDelta::ms(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004311
4312 // Bitrate observer should not be called.
4313 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4314 video_source_.IncomingCapturedFrame(
4315 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4316 ExpectDroppedFrame();
4317
mflodmancc3d4422017-08-03 08:27:51 -07004318 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004319}
ilnik6b826ef2017-06-16 06:53:48 -07004320
Niels Möller4db138e2018-04-19 09:04:13 +02004321TEST_F(VideoStreamEncoderTest,
4322 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4323 const int kFrameWidth = 1280;
4324 const int kFrameHeight = 720;
4325 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004326 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004327 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4328 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004329 video_source_.IncomingCapturedFrame(
4330 CreateFrame(1, kFrameWidth, kFrameHeight));
4331 WaitForEncodedFrame(1);
4332 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4333 .low_encode_usage_threshold_percent,
4334 default_options.low_encode_usage_threshold_percent);
4335 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4336 .high_encode_usage_threshold_percent,
4337 default_options.high_encode_usage_threshold_percent);
4338 video_stream_encoder_->Stop();
4339}
4340
4341TEST_F(VideoStreamEncoderTest,
4342 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4343 const int kFrameWidth = 1280;
4344 const int kFrameHeight = 720;
4345 CpuOveruseOptions hardware_options;
4346 hardware_options.low_encode_usage_threshold_percent = 150;
4347 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004348 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004349
Erik Språng4c6ca302019-04-08 15:14:01 +02004350 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004351 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4352 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004353 video_source_.IncomingCapturedFrame(
4354 CreateFrame(1, kFrameWidth, kFrameHeight));
4355 WaitForEncodedFrame(1);
4356 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4357 .low_encode_usage_threshold_percent,
4358 hardware_options.low_encode_usage_threshold_percent);
4359 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4360 .high_encode_usage_threshold_percent,
4361 hardware_options.high_encode_usage_threshold_percent);
4362 video_stream_encoder_->Stop();
4363}
4364
Niels Möller6bb5ab92019-01-11 11:11:10 +01004365TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4366 const int kFrameWidth = 320;
4367 const int kFrameHeight = 240;
4368 const int kFps = 30;
4369 const int kTargetBitrateBps = 120000;
4370 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4371
Erik Språng4c6ca302019-04-08 15:14:01 +02004372 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004373 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4374 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004375
4376 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4377 max_framerate_ = kFps;
4378
4379 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4380 fake_encoder_.SimulateOvershoot(1.0);
4381 int num_dropped = 0;
4382 for (int i = 0; i < kNumFramesInRun; ++i) {
4383 video_source_.IncomingCapturedFrame(
4384 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4385 // Wait up to two frame durations for a frame to arrive.
4386 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4387 ++num_dropped;
4388 }
4389 timestamp_ms += 1000 / kFps;
4390 }
4391
Erik Språnga8d48ab2019-02-08 14:17:40 +01004392 // Framerate should be measured to be near the expected target rate.
4393 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4394
4395 // Frame drops should be within 5% of expected 0%.
4396 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004397
4398 // Make encoder produce frames at double the expected bitrate during 3 seconds
4399 // of video, verify number of drops. Rate needs to be slightly changed in
4400 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004401 double overshoot_factor = 2.0;
4402 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4403 // With bitrate adjuster, when need to overshoot even more to trigger
4404 // frame dropping.
4405 overshoot_factor *= 2;
4406 }
4407 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004408 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004409 DataRate::bps(kTargetBitrateBps + 1000),
Florent Castellia8336d32019-09-09 13:36:55 +02004410 DataRate::bps(kTargetBitrateBps + 1000),
Erik Språng4c6ca302019-04-08 15:14:01 +02004411 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004412 num_dropped = 0;
4413 for (int i = 0; i < kNumFramesInRun; ++i) {
4414 video_source_.IncomingCapturedFrame(
4415 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4416 // Wait up to two frame durations for a frame to arrive.
4417 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4418 ++num_dropped;
4419 }
4420 timestamp_ms += 1000 / kFps;
4421 }
4422
Erik Språng4c6ca302019-04-08 15:14:01 +02004423 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004424 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4425 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004426
4427 // Target framerate should be still be near the expected target, despite
4428 // the frame drops.
4429 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4430
4431 // Frame drops should be within 5% of expected 50%.
4432 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004433
4434 video_stream_encoder_->Stop();
4435}
4436
4437TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4438 const int kFrameWidth = 320;
4439 const int kFrameHeight = 240;
4440 const int kActualInputFps = 24;
4441 const int kTargetBitrateBps = 120000;
4442
4443 ASSERT_GT(max_framerate_, kActualInputFps);
4444
4445 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4446 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004447 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004448 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4449 DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004450
4451 // Insert 3 seconds of video, with an input fps lower than configured max.
4452 for (int i = 0; i < kActualInputFps * 3; ++i) {
4453 video_source_.IncomingCapturedFrame(
4454 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4455 // Wait up to two frame durations for a frame to arrive.
4456 WaitForEncodedFrame(timestamp_ms);
4457 timestamp_ms += 1000 / kActualInputFps;
4458 }
4459
4460 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4461
4462 video_stream_encoder_->Stop();
4463}
4464
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004465TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4466 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004467 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004468 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4469 DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004470
4471 fake_encoder_.BlockNextEncode();
4472 video_source_.IncomingCapturedFrame(
4473 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4474 WaitForEncodedFrame(1);
4475 // On the very first frame full update should be forced.
4476 rect = fake_encoder_.GetLastUpdateRect();
4477 EXPECT_EQ(rect.offset_x, 0);
4478 EXPECT_EQ(rect.offset_y, 0);
4479 EXPECT_EQ(rect.height, codec_height_);
4480 EXPECT_EQ(rect.width, codec_width_);
4481 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4482 // call to ContinueEncode.
4483 video_source_.IncomingCapturedFrame(
4484 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4485 ExpectDroppedFrame();
4486 video_source_.IncomingCapturedFrame(
4487 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4488 ExpectDroppedFrame();
4489 fake_encoder_.ContinueEncode();
4490 WaitForEncodedFrame(3);
4491 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4492 rect = fake_encoder_.GetLastUpdateRect();
4493 EXPECT_EQ(rect.offset_x, 1);
4494 EXPECT_EQ(rect.offset_y, 0);
4495 EXPECT_EQ(rect.width, 10);
4496 EXPECT_EQ(rect.height, 1);
4497
4498 video_source_.IncomingCapturedFrame(
4499 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4500 WaitForEncodedFrame(4);
4501 // Previous frame was encoded, so no accumulation should happen.
4502 rect = fake_encoder_.GetLastUpdateRect();
4503 EXPECT_EQ(rect.offset_x, 0);
4504 EXPECT_EQ(rect.offset_y, 0);
4505 EXPECT_EQ(rect.width, 1);
4506 EXPECT_EQ(rect.height, 1);
4507
4508 video_stream_encoder_->Stop();
4509}
4510
Erik Språngd7329ca2019-02-21 21:19:53 +01004511TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004512 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004513 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4514 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004515
4516 // First frame is always keyframe.
4517 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4518 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004519 EXPECT_THAT(
4520 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004521 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004522
4523 // Insert delta frame.
4524 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4525 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01004526 EXPECT_THAT(
4527 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004528 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004529
4530 // Request next frame be a key-frame.
4531 video_stream_encoder_->SendKeyFrame();
4532 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4533 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01004534 EXPECT_THAT(
4535 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004536 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004537
4538 video_stream_encoder_->Stop();
4539}
4540
4541TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4542 // Setup simulcast with three streams.
4543 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01004544 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004545 DataRate::bps(kSimulcastTargetBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02004546 DataRate::bps(kSimulcastTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02004547 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004548 // Wait for all three layers before triggering event.
4549 sink_.SetNumExpectedLayers(3);
4550
4551 // First frame is always keyframe.
4552 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4553 WaitForEncodedFrame(1);
4554 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004555 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4556 VideoFrameType::kVideoFrameKey,
4557 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004558
4559 // Insert delta frame.
4560 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4561 WaitForEncodedFrame(2);
4562 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004563 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4564 VideoFrameType::kVideoFrameDelta,
4565 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004566
4567 // Request next frame be a key-frame.
4568 // Only first stream is configured to produce key-frame.
4569 video_stream_encoder_->SendKeyFrame();
4570 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4571 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02004572
4573 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4574 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01004575 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004576 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02004577 VideoFrameType::kVideoFrameKey,
4578 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004579
4580 video_stream_encoder_->Stop();
4581}
4582
4583TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
4584 // Configure internal source factory and setup test again.
4585 encoder_factory_.SetHasInternalSource(true);
4586 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004587 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004588 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4589 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004590
4591 // Call encoder directly, simulating internal source where encoded frame
4592 // callback in VideoStreamEncoder is called despite no OnFrame().
4593 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
4594 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004595 EXPECT_THAT(
4596 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004597 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004598
Niels Möller8f7ce222019-03-21 15:43:58 +01004599 const std::vector<VideoFrameType> kDeltaFrame = {
4600 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01004601 // Need to set timestamp manually since manually for injected frame.
4602 VideoFrame frame = CreateFrame(101, nullptr);
4603 frame.set_timestamp(101);
4604 fake_encoder_.InjectFrame(frame, false);
4605 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004606 EXPECT_THAT(
4607 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004608 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004609
4610 // Request key-frame. The forces a dummy frame down into the encoder.
4611 fake_encoder_.ExpectNullFrame();
4612 video_stream_encoder_->SendKeyFrame();
4613 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004614 EXPECT_THAT(
4615 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004616 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004617
4618 video_stream_encoder_->Stop();
4619}
Erik Språngb7cb7b52019-02-26 15:52:33 +01004620
4621TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
4622 // Configure internal source factory and setup test again.
4623 encoder_factory_.SetHasInternalSource(true);
4624 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004625 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004626 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4627 DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01004628
4629 int64_t timestamp = 1;
4630 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02004631 image.SetEncodedData(
4632 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01004633 image.capture_time_ms_ = ++timestamp;
4634 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
4635 const int64_t kEncodeFinishDelayMs = 10;
4636 image.timing_.encode_start_ms = timestamp;
4637 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
4638 fake_encoder_.InjectEncodedImage(image);
4639 // Wait for frame without incrementing clock.
4640 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4641 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
4642 // capture timestamp should be kEncodeFinishDelayMs in the past.
4643 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
4644 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
4645 kEncodeFinishDelayMs);
4646
4647 video_stream_encoder_->Stop();
4648}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02004649
4650TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
4651 // Configure internal source factory and setup test again.
4652 encoder_factory_.SetHasInternalSource(true);
4653 ResetEncoder("H264", 1, 1, 1, false);
4654
4655 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
4656 image._frameType = VideoFrameType::kVideoFrameKey;
4657
4658 CodecSpecificInfo codec_specific_info;
4659 codec_specific_info.codecType = kVideoCodecH264;
4660
4661 RTPFragmentationHeader fragmentation;
4662 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4663 fragmentation.fragmentationOffset[0] = 4;
4664 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
4665
4666 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4667 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4668
4669 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4670 testing::ElementsAreArray(optimal_sps));
4671 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4672 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4673 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4674 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4675
4676 video_stream_encoder_->Stop();
4677}
4678
4679TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
4680 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
4681 0x00, 0x00, 0x03, 0x03, 0xF4,
4682 0x05, 0x03, 0xC7, 0xC0};
4683
4684 // Configure internal source factory and setup test again.
4685 encoder_factory_.SetHasInternalSource(true);
4686 ResetEncoder("H264", 1, 1, 1, false);
4687
4688 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
4689 image._frameType = VideoFrameType::kVideoFrameKey;
4690
4691 CodecSpecificInfo codec_specific_info;
4692 codec_specific_info.codecType = kVideoCodecH264;
4693
4694 RTPFragmentationHeader fragmentation;
4695 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4696 fragmentation.fragmentationOffset[0] = 4;
4697 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
4698
4699 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4700 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4701
4702 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4703 testing::ElementsAreArray(optimal_sps));
4704 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4705 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4706 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4707 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4708
4709 video_stream_encoder_->Stop();
4710}
4711
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02004712TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
4713 const int kFrameWidth = 1280;
4714 const int kFrameHeight = 720;
4715 const int kTargetBitrateBps = 300000; // To low for HD resolution.
4716
4717 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004718 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
4719 DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02004720 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4721
4722 // Insert a first video frame. It should be dropped because of downscale in
4723 // resolution.
4724 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4725 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4726 frame.set_rotation(kVideoRotation_270);
4727 video_source_.IncomingCapturedFrame(frame);
4728
4729 ExpectDroppedFrame();
4730
4731 // Second frame is downscaled.
4732 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4733 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4734 frame.set_rotation(kVideoRotation_90);
4735 video_source_.IncomingCapturedFrame(frame);
4736
4737 WaitForEncodedFrame(timestamp_ms);
4738 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
4739
4740 // Insert another frame, also downscaled.
4741 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4742 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4743 frame.set_rotation(kVideoRotation_180);
4744 video_source_.IncomingCapturedFrame(frame);
4745
4746 WaitForEncodedFrame(timestamp_ms);
4747 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
4748
4749 video_stream_encoder_->Stop();
4750}
4751
Erik Språng5056af02019-09-02 15:53:11 +02004752TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
4753 const int kFrameWidth = 320;
4754 const int kFrameHeight = 180;
4755
4756 // Initial rate.
4757 video_stream_encoder_->OnBitrateUpdated(
4758 /*target_bitrate=*/DataRate::kbps(300),
Florent Castellia8336d32019-09-09 13:36:55 +02004759 /*stable_target_bitrate=*/DataRate::kbps(300),
Erik Språng5056af02019-09-02 15:53:11 +02004760 /*link_allocation=*/DataRate::kbps(300),
4761 /*fraction_lost=*/0,
4762 /*rtt_ms=*/0);
4763
4764 // Insert a first video frame so that encoder gets configured.
4765 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4766 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4767 frame.set_rotation(kVideoRotation_270);
4768 video_source_.IncomingCapturedFrame(frame);
4769 WaitForEncodedFrame(timestamp_ms);
4770
4771 // Set a target rate below the minimum allowed by the codec settings.
4772 VideoCodec codec_config = fake_encoder_.codec_config();
4773 DataRate min_rate = DataRate::kbps(codec_config.minBitrate);
4774 DataRate target_rate = min_rate - DataRate::kbps(1);
4775 video_stream_encoder_->OnBitrateUpdated(
4776 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02004777 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02004778 /*link_allocation=*/target_rate,
4779 /*fraction_lost=*/0,
4780 /*rtt_ms=*/0);
4781 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4782
4783 // Target bitrate and bandwidth allocation should both be capped at min_rate.
4784 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
4785 ASSERT_TRUE(rate_settings.has_value());
4786 DataRate allocation_sum = DataRate::bps(rate_settings->bitrate.get_sum_bps());
4787 EXPECT_EQ(min_rate, allocation_sum);
4788 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
4789
4790 video_stream_encoder_->Stop();
4791}
4792
philipeld9cc8c02019-09-16 14:53:40 +02004793struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
4794 MOCK_METHOD0(RequestEncoderFallback, void());
4795 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
4796};
4797
4798TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
4799 constexpr int kDontCare = 100;
4800
4801 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
4802 video_send_config_.encoder_settings.encoder_switch_request_callback =
4803 &switch_callback;
4804 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
4805 encoder_config.codec_type = kVideoCodecVP8;
4806 webrtc::test::ScopedFieldTrials field_trial(
4807 "WebRTC-NetworkCondition-EncoderSwitch/"
4808 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
4809 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
4810
4811 // Reset encoder for new configuration to take effect.
4812 ConfigureEncoder(std::move(encoder_config));
4813
4814 // Send one frame to trigger ReconfigureEncoder.
4815 video_source_.IncomingCapturedFrame(
4816 CreateFrame(kDontCare, kDontCare, kDontCare));
4817
4818 using Config = EncoderSwitchRequestCallback::Config;
4819 EXPECT_CALL(switch_callback,
4820 RequestEncoderSwitch(AllOf(Field(&Config::codec_name, "AV1"),
4821 Field(&Config::param, "ping"),
4822 Field(&Config::value, "pong"))));
4823
4824 video_stream_encoder_->OnBitrateUpdated(
4825 /*target_bitrate=*/DataRate::kbps(50),
4826 /*stable_target_bitrate=*/DataRate::kbps(kDontCare),
4827 /*link_allocation=*/DataRate::kbps(kDontCare),
4828 /*fraction_lost=*/0,
4829 /*rtt_ms=*/0);
4830
4831 video_stream_encoder_->Stop();
4832}
4833
4834TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
4835 constexpr int kSufficientBitrateToNotDrop = 1000;
4836 constexpr int kHighRes = 500;
4837 constexpr int kLowRes = 100;
4838
4839 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
4840 video_send_config_.encoder_settings.encoder_switch_request_callback =
4841 &switch_callback;
4842 webrtc::test::ScopedFieldTrials field_trial(
4843 "WebRTC-NetworkCondition-EncoderSwitch/"
4844 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
4845 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
4846 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
4847 encoder_config.codec_type = kVideoCodecH264;
4848
4849 // Reset encoder for new configuration to take effect.
4850 ConfigureEncoder(std::move(encoder_config));
4851
4852 // The VideoStreamEncoder needs some bitrate before it can start encoding,
4853 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
4854 // not fail.
4855 video_stream_encoder_->OnBitrateUpdated(
4856 /*target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
4857 /*stable_target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
4858 /*link_allocation=*/DataRate::kbps(kSufficientBitrateToNotDrop),
4859 /*fraction_lost=*/0,
4860 /*rtt_ms=*/0);
4861
4862 // Send one frame to trigger ReconfigureEncoder.
4863 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
4864 WaitForEncodedFrame(1);
4865
4866 using Config = EncoderSwitchRequestCallback::Config;
4867 EXPECT_CALL(switch_callback,
4868 RequestEncoderSwitch(AllOf(Field(&Config::codec_name, "AV1"),
4869 Field(&Config::param, "ping"),
4870 Field(&Config::value, "pong"))));
4871
4872 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
4873 WaitForEncodedFrame(2);
4874
4875 video_stream_encoder_->Stop();
4876}
4877
perkj26091b12016-09-01 01:17:40 -07004878} // namespace webrtc