blob: d20b5ab89108c1a23083a9d5cc2cad736eab55af [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010022#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010023#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020024#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010026#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010027#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080028#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020030#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020031#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010032#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010033#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020034#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020035#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010036#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020037#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010038#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020039#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070040#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020042#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 18:47:52 +010043#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010044#include "modules/video_coding/codecs/h264/include/h264.h"
45#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
46#include "modules/video_coding/codecs/vp8/include/vp8.h"
47#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020048#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020049#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020050#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010051#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010053#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020054#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080056#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020057#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020058#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/encoder_settings.h"
60#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010061#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/gmock.h"
63#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010064#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010065#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020066#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020067#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010068#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020069#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070070
71namespace webrtc {
72
sprang57c2fff2017-01-16 06:24:02 -080073using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020074using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020075using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Ge;
78using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010079using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020080using ::testing::Le;
81using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010082using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010083using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010085using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010087using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020088using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080089
perkj803d97f2016-11-01 11:45:46 -070090namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020091const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010092const int kQpLow = 1;
93const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020094const int kMinFramerateFps = 2;
95const int kMinBalancedFramerateFps = 7;
96const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080097const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020098const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
99const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
100const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
101const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800102const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700103const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200104const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200105const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800110
Asa Persson606d3cb2021-10-04 10:07:11 +0200111uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200112 0x00, 0x00, 0x03, 0x03, 0xF4,
113 0x05, 0x03, 0xC7, 0xE0, 0x1B,
114 0x41, 0x10, 0x8D, 0x00};
115
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100116const uint8_t kCodedFrameVp8Qp25[] = {
117 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
118 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
119 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
120
Markus Handell818e7fb2021-12-30 13:01:33 +0100121VideoFrame CreateSimpleNV12Frame() {
122 return VideoFrame::Builder()
123 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
124 /*width=*/16, /*height=*/16))
125 .build();
126}
127
Markus Handell8d87c462021-12-16 11:37:16 +0100128void PassAFrame(
129 TaskQueueBase* encoder_queue,
130 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
131 int64_t ntp_time_ms) {
132 encoder_queue->PostTask(
133 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 13:01:33 +0100134 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
135 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 11:37:16 +0100136 }));
137}
138
perkj803d97f2016-11-01 11:45:46 -0700139class TestBuffer : public webrtc::I420Buffer {
140 public:
141 TestBuffer(rtc::Event* event, int width, int height)
142 : I420Buffer(width, height), event_(event) {}
143
144 private:
145 friend class rtc::RefCountedObject<TestBuffer>;
146 ~TestBuffer() override {
147 if (event_)
148 event_->Set();
149 }
150 rtc::Event* const event_;
151};
152
Henrik Boström56db9ff2021-03-24 09:06:45 +0100153// A fake native buffer that can't be converted to I420. Upon scaling, it
154// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700155class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
156 public:
157 FakeNativeBuffer(rtc::Event* event, int width, int height)
158 : event_(event), width_(width), height_(height) {}
159 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
160 int width() const override { return width_; }
161 int height() const override { return height_; }
162 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
163 return nullptr;
164 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100165 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
166 int offset_x,
167 int offset_y,
168 int crop_width,
169 int crop_height,
170 int scaled_width,
171 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200172 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
173 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100174 }
Noah Richards51db4212019-06-12 06:59:12 -0700175
176 private:
177 friend class rtc::RefCountedObject<FakeNativeBuffer>;
178 ~FakeNativeBuffer() override {
179 if (event_)
180 event_->Set();
181 }
182 rtc::Event* const event_;
183 const int width_;
184 const int height_;
185};
186
Evan Shrubsole895556e2020-10-05 09:15:13 +0200187// A fake native buffer that is backed by an NV12 buffer.
188class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
189 public:
190 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
191 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
192
193 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
194 int width() const override { return nv12_buffer_->width(); }
195 int height() const override { return nv12_buffer_->height(); }
196 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
197 return nv12_buffer_->ToI420();
198 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200199 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
200 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
201 if (absl::c_find(types, Type::kNV12) != types.end()) {
202 return nv12_buffer_;
203 }
204 return nullptr;
205 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200206 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
207
208 private:
209 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
210 ~FakeNV12NativeBuffer() override {
211 if (event_)
212 event_->Set();
213 }
214 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
215 rtc::Event* const event_;
216};
217
Niels Möller7dc26b72017-12-06 10:27:48 +0100218class CpuOveruseDetectorProxy : public OveruseFrameDetector {
219 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200220 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
221 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200222 last_target_framerate_fps_(-1),
223 framerate_updated_event_(true /* manual_reset */,
224 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100225 virtual ~CpuOveruseDetectorProxy() {}
226
227 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200228 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 last_target_framerate_fps_ = framerate_fps;
230 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200231 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100232 }
233
234 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200235 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100236 return last_target_framerate_fps_;
237 }
238
Niels Möller4db138e2018-04-19 09:04:13 +0200239 CpuOveruseOptions GetOptions() { return options_; }
240
Henrik Boström381d1092020-05-12 18:49:07 +0200241 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
242
Niels Möller7dc26b72017-12-06 10:27:48 +0100243 private:
Markus Handella3765182020-07-08 13:13:32 +0200244 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100245 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200246 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100247};
248
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200249class FakeVideoSourceRestrictionsListener
250 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200251 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200252 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200253 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200254 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200255 RTC_DCHECK(was_restrictions_updated_);
256 }
257
258 rtc::Event* restrictions_updated_event() {
259 return &restrictions_updated_event_;
260 }
261
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200262 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200263 void OnVideoSourceRestrictionsUpdated(
264 VideoSourceRestrictions restrictions,
265 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200266 rtc::scoped_refptr<Resource> reason,
267 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200268 was_restrictions_updated_ = true;
269 restrictions_updated_event_.Set();
270 }
271
272 private:
273 bool was_restrictions_updated_;
274 rtc::Event restrictions_updated_event_;
275};
276
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200277auto WantsFps(Matcher<int> fps_matcher) {
278 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
279 fps_matcher);
280}
281
282auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
283 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
284 AllOf(max_pixel_matcher, Gt(0)));
285}
286
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200287auto ResolutionMax() {
288 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200289 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200290 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
291 Eq(absl::nullopt)));
292}
293
294auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200295 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200296}
297
298auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200299 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300}
301
302auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200303 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304}
305
306auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200307 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200308}
309
310auto FpsMaxResolutionMax() {
311 return AllOf(FpsMax(), ResolutionMax());
312}
313
314auto UnlimitedSinkWants() {
315 return AllOf(FpsUnlimited(), ResolutionMax());
316}
317
318auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
319 Matcher<int> fps_range_matcher;
320
321 if (last_frame_pixels <= 320 * 240) {
322 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200323 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200324 fps_range_matcher = AllOf(Ge(10), Le(15));
325 } else if (last_frame_pixels <= 640 * 480) {
326 fps_range_matcher = Ge(15);
327 } else {
328 fps_range_matcher = Eq(kDefaultFramerate);
329 }
330 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
331 fps_range_matcher);
332}
333
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200334auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
335 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
336 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
337}
338
339auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
340 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
341}
342
343auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
345}
346
347auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
348 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
349 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
350}
351
352auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
353 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
354 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
355}
356
357auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
358 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
359 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
360}
361
362auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
363 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
364 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
365}
366
mflodmancc3d4422017-08-03 08:27:51 -0700367class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700368 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100369 VideoStreamEncoderUnderTest(
370 TimeController* time_controller,
371 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
372 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
373 encoder_queue,
374 SendStatisticsProxy* stats_proxy,
375 const VideoStreamEncoderSettings& settings,
376 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100377 allocation_callback_type,
378 const WebRtcKeyValueConfig& field_trials)
Markus Handell9a478b52021-11-18 16:07:01 +0100379 : VideoStreamEncoder(time_controller->GetClock(),
380 1 /* number_of_cores */,
381 stats_proxy,
382 settings,
383 std::unique_ptr<OveruseFrameDetector>(
384 overuse_detector_proxy_ =
385 new CpuOveruseDetectorProxy(stats_proxy)),
386 std::move(cadence_adapter),
387 std::move(encoder_queue),
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100388 allocation_callback_type,
389 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200390 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200391 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200392 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200393 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200394 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200395 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200396 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200397 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100398 }
perkj803d97f2016-11-01 11:45:46 -0700399
Henrik Boström381d1092020-05-12 18:49:07 +0200400 void SetSourceAndWaitForRestrictionsUpdated(
401 rtc::VideoSourceInterface<VideoFrame>* source,
402 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200403 FakeVideoSourceRestrictionsListener listener;
404 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200405 SetSource(source, degradation_preference);
406 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200407 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200408 }
409
410 void SetSourceAndWaitForFramerateUpdated(
411 rtc::VideoSourceInterface<VideoFrame>* source,
412 const DegradationPreference& degradation_preference) {
413 overuse_detector_proxy_->framerate_updated_event()->Reset();
414 SetSource(source, degradation_preference);
415 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
416 }
417
418 void OnBitrateUpdatedAndWaitForManagedResources(
419 DataRate target_bitrate,
420 DataRate stable_target_bitrate,
421 DataRate link_allocation,
422 uint8_t fraction_lost,
423 int64_t round_trip_time_ms,
424 double cwnd_reduce_ratio) {
425 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
426 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
427 // Bitrate is updated on the encoder queue.
428 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200429 }
430
kthelgason2fc52542017-03-03 00:24:41 -0800431 // This is used as a synchronisation mechanism, to make sure that the
432 // encoder queue is not blocked before we start sending it frames.
433 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100434 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800435 }
436
Henrik Boström91aa7322020-04-28 12:24:33 +0200437 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200438 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200439 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200440 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200441 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200442 event.Set();
443 });
444 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100445 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 void TriggerCpuUnderuse() {
449 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200450 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200451 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200452 event.Set();
453 });
454 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100455 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200456 }
kthelgason876222f2016-11-29 01:44:11 -0800457
Henrik Boström91aa7322020-04-28 12:24:33 +0200458 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200459 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200460 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200461 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200462 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200463 event.Set();
464 });
465 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100466 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200467 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200468 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200470 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200471 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 event.Set();
473 });
474 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100475 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200476 }
477
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200478 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100479 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200480 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
481 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200482 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700483};
484
Noah Richards51db4212019-06-12 06:59:12 -0700485// Simulates simulcast behavior and makes highest stream resolutions divisible
486// by 4.
487class CroppingVideoStreamFactory
488 : public VideoEncoderConfig::VideoStreamFactoryInterface {
489 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200490 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700491
492 private:
493 std::vector<VideoStream> CreateEncoderStreams(
494 int width,
495 int height,
496 const VideoEncoderConfig& encoder_config) override {
497 std::vector<VideoStream> streams = test::CreateVideoStreams(
498 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700499 return streams;
500 }
Noah Richards51db4212019-06-12 06:59:12 -0700501};
502
sprangb1ca0732017-02-01 08:38:12 -0800503class AdaptingFrameForwarder : public test::FrameForwarder {
504 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200505 explicit AdaptingFrameForwarder(TimeController* time_controller)
506 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700507 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800508
509 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200510 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800511 adaptation_enabled_ = enabled;
512 }
513
asaperssonfab67072017-04-04 05:51:49 -0700514 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200515 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800516 return adaptation_enabled_;
517 }
518
Henrik Boström1124ed12021-02-25 10:30:39 +0100519 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
520 // the resolution or frame rate was different than it is currently. If
521 // something else is modified, such as encoder resolutions, but the resolution
522 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700523 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200524 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700525 return last_wants_;
526 }
527
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200528 absl::optional<int> last_sent_width() const { return last_width_; }
529 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800530
sprangb1ca0732017-02-01 08:38:12 -0800531 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200532 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100533 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200534
sprangb1ca0732017-02-01 08:38:12 -0800535 int cropped_width = 0;
536 int cropped_height = 0;
537 int out_width = 0;
538 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700539 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000540 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
541 << "w=" << video_frame.width()
542 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700543 if (adapter_.AdaptFrameResolution(
544 video_frame.width(), video_frame.height(),
545 video_frame.timestamp_us() * 1000, &cropped_width,
546 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100547 VideoFrame adapted_frame =
548 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200549 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100550 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200551 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100552 .set_timestamp_ms(99)
553 .set_rotation(kVideoRotation_0)
554 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100555 if (video_frame.has_update_rect()) {
556 adapted_frame.set_update_rect(
557 video_frame.update_rect().ScaleWithFrame(
558 video_frame.width(), video_frame.height(), 0, 0,
559 video_frame.width(), video_frame.height(), out_width,
560 out_height));
561 }
sprangc5d62e22017-04-02 23:53:04 -0700562 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800563 last_width_.emplace(adapted_frame.width());
564 last_height_.emplace(adapted_frame.height());
565 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200566 last_width_ = absl::nullopt;
567 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700568 }
sprangb1ca0732017-02-01 08:38:12 -0800569 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000570 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800571 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800572 last_width_.emplace(video_frame.width());
573 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800574 }
575 }
576
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200577 void OnOutputFormatRequest(int width, int height) {
578 absl::optional<std::pair<int, int>> target_aspect_ratio =
579 std::make_pair(width, height);
580 absl::optional<int> max_pixel_count = width * height;
581 absl::optional<int> max_fps;
582 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
583 max_fps);
584 }
585
sprangb1ca0732017-02-01 08:38:12 -0800586 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
587 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200588 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100589 rtc::VideoSinkWants prev_wants = sink_wants_locked();
590 bool did_adapt =
591 prev_wants.max_pixel_count != wants.max_pixel_count ||
592 prev_wants.target_pixel_count != wants.target_pixel_count ||
593 prev_wants.max_framerate_fps != wants.max_framerate_fps;
594 if (did_adapt) {
595 last_wants_ = prev_wants;
596 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100597 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200598 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800599 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200600
601 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800602 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200603 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
604 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200605 absl::optional<int> last_width_;
606 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800607};
sprangc5d62e22017-04-02 23:53:04 -0700608
Niels Möller213618e2018-07-24 09:29:58 +0200609// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700610class MockableSendStatisticsProxy : public SendStatisticsProxy {
611 public:
612 MockableSendStatisticsProxy(Clock* clock,
613 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100614 VideoEncoderConfig::ContentType content_type,
615 const WebRtcKeyValueConfig& field_trials)
616 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700617
618 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200619 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700620 if (mock_stats_)
621 return *mock_stats_;
622 return SendStatisticsProxy::GetStats();
623 }
624
Niels Möller213618e2018-07-24 09:29:58 +0200625 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200626 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200627 if (mock_stats_)
628 return mock_stats_->input_frame_rate;
629 return SendStatisticsProxy::GetInputFrameRate();
630 }
sprangc5d62e22017-04-02 23:53:04 -0700631 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200632 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700633 mock_stats_.emplace(stats);
634 }
635
636 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200637 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700638 mock_stats_.reset();
639 }
640
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200641 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
642 on_frame_dropped_ = std::move(callback);
643 }
644
sprangc5d62e22017-04-02 23:53:04 -0700645 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200646 void OnFrameDropped(DropReason reason) override {
647 SendStatisticsProxy::OnFrameDropped(reason);
648 if (on_frame_dropped_)
649 on_frame_dropped_(reason);
650 }
651
Markus Handella3765182020-07-08 13:13:32 +0200652 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200653 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200654 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700655};
656
Markus Handellb4e96d42021-11-05 12:00:55 +0100657class SimpleVideoStreamEncoderFactory {
658 public:
659 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
660 public:
661 using VideoStreamEncoder::VideoStreamEncoder;
662 ~AdaptedVideoStreamEncoder() { Stop(); }
663 };
664
Markus Handell8d87c462021-12-16 11:37:16 +0100665 class MockFakeEncoder : public test::FakeEncoder {
666 public:
667 using FakeEncoder::FakeEncoder;
668 MOCK_METHOD(CodecSpecificInfo,
669 EncodeHook,
670 (EncodedImage & encoded_image,
671 rtc::scoped_refptr<EncodedImageBuffer> buffer),
672 (override));
673 };
674
Markus Handellee225432021-11-29 12:35:12 +0100675 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100676 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100677 encoder_settings_.bitrate_allocator_factory =
678 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100679 }
680
Markus Handell818e7fb2021-12-30 13:01:33 +0100681 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100682 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100683 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
684 const WebRtcKeyValueConfig* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100685 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
686 time_controller_.GetClock(),
687 /*number_of_cores=*/1,
688 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
689 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100690 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100691 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100692 kVideoBitrateAllocation,
693 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100694 result->SetSink(&sink_, /*rotation_applied=*/false);
695 return result;
696 }
697
Markus Handell818e7fb2021-12-30 13:01:33 +0100698 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
699 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
700 TaskQueueBase** encoder_queue_ptr = nullptr) {
701 auto encoder_queue =
702 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
703 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
704 if (encoder_queue_ptr)
705 *encoder_queue_ptr = encoder_queue.get();
706 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
707 std::move(encoder_queue));
708 }
709
Markus Handell9a478b52021-11-18 16:07:01 +0100710 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100711 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100712
Markus Handell818e7fb2021-12-30 13:01:33 +0100713 GlobalSimulatedTimeController* GetTimeController() {
714 return &time_controller_;
715 }
716
Markus Handellb4e96d42021-11-05 12:00:55 +0100717 private:
718 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
719 public:
720 ~NullEncoderSink() override = default;
721 void OnEncoderConfigurationChanged(
722 std::vector<VideoStream> streams,
723 bool is_svc,
724 VideoEncoderConfig::ContentType content_type,
725 int min_transmit_bitrate_bps) override {}
726 void OnBitrateAllocationUpdated(
727 const VideoBitrateAllocation& allocation) override {}
728 void OnVideoLayersAllocationUpdated(
729 VideoLayersAllocation allocation) override {}
730 Result OnEncodedImage(
731 const EncodedImage& encoded_image,
732 const CodecSpecificInfo* codec_specific_info) override {
733 return Result(EncodedImageCallback::Result::OK);
734 }
735 };
736
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100737 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100738 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
739 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
740 time_controller_.CreateTaskQueueFactory()};
741 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
742 std::make_unique<MockableSendStatisticsProxy>(
743 time_controller_.GetClock(),
744 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100745 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
746 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100747 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
748 CreateBuiltinVideoBitrateAllocatorFactory();
749 VideoStreamEncoderSettings encoder_settings_{
750 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100751 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
752 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100753 NullEncoderSink sink_;
754};
755
756class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
757 public:
758 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100759 MOCK_METHOD(void,
760 SetZeroHertzModeEnabled,
761 (absl::optional<ZeroHertzModeParams>),
762 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100763 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100764 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
765 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100766 MOCK_METHOD(void,
767 UpdateLayerQualityConvergence,
768 (int spatial_index, bool converged),
769 (override));
770 MOCK_METHOD(void,
771 UpdateLayerStatus,
772 (int spatial_index, bool enabled),
773 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100774 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100775};
776
philipel9b058032020-02-10 11:30:00 +0100777class MockEncoderSelector
778 : public VideoEncoderFactory::EncoderSelectorInterface {
779 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200780 MOCK_METHOD(void,
781 OnCurrentEncoder,
782 (const SdpVideoFormat& format),
783 (override));
784 MOCK_METHOD(absl::optional<SdpVideoFormat>,
785 OnAvailableBitrate,
786 (const DataRate& rate),
787 (override));
788 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100789};
790
Markus Handell2e0f4f02021-12-21 19:14:58 +0100791class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
792 public:
793 MOCK_METHOD(void,
794 AddOrUpdateSink,
795 (rtc::VideoSinkInterface<VideoFrame>*,
796 const rtc::VideoSinkWants&),
797 (override));
798 MOCK_METHOD(void,
799 RemoveSink,
800 (rtc::VideoSinkInterface<VideoFrame>*),
801 (override));
802 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
803};
804
perkj803d97f2016-11-01 11:45:46 -0700805} // namespace
806
mflodmancc3d4422017-08-03 08:27:51 -0700807class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700808 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200809 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700810
mflodmancc3d4422017-08-03 08:27:51 -0700811 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700812 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700813 codec_width_(320),
814 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200815 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200816 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200817 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700818 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200819 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700820 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100821 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
822 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200823 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700824
825 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700826 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700827 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200828 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800829 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200830 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200831 video_send_config_.rtp.payload_name = "FAKE";
832 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700833
Per512ecb32016-09-23 15:52:06 +0200834 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200835 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200836 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
837 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
838 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100839 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700840
Niels Möllerf1338562018-04-26 09:51:47 +0200841 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800842 }
843
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100844 void ConfigureEncoder(
845 VideoEncoderConfig video_encoder_config,
846 VideoStreamEncoder::BitrateAllocationCallbackType
847 allocation_callback_type =
848 VideoStreamEncoder::BitrateAllocationCallbackType::
849 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700850 if (video_stream_encoder_)
851 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100852
853 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
854 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
855 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
856 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
857 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100858 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100859 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
860 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
861 stats_proxy_.get(), video_send_config_.encoder_settings,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100862 allocation_callback_type, field_trials_);
Asa Persson606d3cb2021-10-04 10:07:11 +0200863 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700864 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700865 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200866 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700867 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200868 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700869 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800870 }
871
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100872 void ResetEncoder(const std::string& payload_name,
873 size_t num_streams,
874 size_t num_temporal_layers,
875 unsigned char num_spatial_layers,
876 bool screenshare,
877 VideoStreamEncoder::BitrateAllocationCallbackType
878 allocation_callback_type =
879 VideoStreamEncoder::BitrateAllocationCallbackType::
880 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200881 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800882
883 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200884 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
885 num_streams, &video_encoder_config);
886 for (auto& layer : video_encoder_config.simulcast_layers) {
887 layer.num_temporal_layers = num_temporal_layers;
888 layer.max_framerate = kDefaultFramerate;
889 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100890 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200891 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700892 video_encoder_config.content_type =
893 screenshare ? VideoEncoderConfig::ContentType::kScreen
894 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700895 if (payload_name == "VP9") {
896 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
897 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200898 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700899 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200900 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
901 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700902 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100903 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700904 }
905
sprang57c2fff2017-01-16 06:24:02 -0800906 VideoFrame CreateFrame(int64_t ntp_time_ms,
907 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200908 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200909 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200910 destruction_event, codec_width_, codec_height_))
911 .set_ntp_time_ms(ntp_time_ms)
912 .set_timestamp_ms(99)
913 .set_rotation(kVideoRotation_0)
914 .build();
perkj26091b12016-09-01 01:17:40 -0700915 }
916
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100917 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
918 rtc::Event* destruction_event,
919 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200920 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200921 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200922 destruction_event, codec_width_, codec_height_))
923 .set_ntp_time_ms(ntp_time_ms)
924 .set_timestamp_ms(99)
925 .set_rotation(kVideoRotation_0)
926 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
927 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100928 }
929
sprang57c2fff2017-01-16 06:24:02 -0800930 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200931 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
932 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200933 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200934 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200935 .set_ntp_time_ms(ntp_time_ms)
936 .set_timestamp_ms(ntp_time_ms)
937 .set_rotation(kVideoRotation_0)
938 .build();
perkj803d97f2016-11-01 11:45:46 -0700939 }
940
Evan Shrubsole895556e2020-10-05 09:15:13 +0200941 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200942 return VideoFrame::Builder()
943 .set_video_frame_buffer(NV12Buffer::Create(width, height))
944 .set_ntp_time_ms(ntp_time_ms)
945 .set_timestamp_ms(ntp_time_ms)
946 .set_rotation(kVideoRotation_0)
947 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200948 }
949
Noah Richards51db4212019-06-12 06:59:12 -0700950 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
951 rtc::Event* destruction_event,
952 int width,
953 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200954 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200955 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200956 destruction_event, width, height))
957 .set_ntp_time_ms(ntp_time_ms)
958 .set_timestamp_ms(99)
959 .set_rotation(kVideoRotation_0)
960 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700961 }
962
Evan Shrubsole895556e2020-10-05 09:15:13 +0200963 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
964 rtc::Event* destruction_event,
965 int width,
966 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200967 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200968 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200969 destruction_event, width, height))
970 .set_ntp_time_ms(ntp_time_ms)
971 .set_timestamp_ms(99)
972 .set_rotation(kVideoRotation_0)
973 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200974 }
975
Noah Richards51db4212019-06-12 06:59:12 -0700976 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
977 rtc::Event* destruction_event) const {
978 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
979 codec_height_);
980 }
981
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100982 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200984 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100985
986 video_source_.IncomingCapturedFrame(
987 CreateFrame(1, codec_width_, codec_height_));
988 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200989 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100990 }
991
sprang4847ae62017-06-27 07:06:52 -0700992 void WaitForEncodedFrame(int64_t expected_ntp_time) {
993 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200994 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700995 }
996
997 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
998 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200999 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001000 return ok;
1001 }
1002
1003 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1004 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001005 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001006 }
1007
1008 void ExpectDroppedFrame() {
1009 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001010 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001011 }
1012
1013 bool WaitForFrame(int64_t timeout_ms) {
1014 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001015 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001016 return ok;
1017 }
1018
perkj26091b12016-09-01 01:17:40 -07001019 class TestEncoder : public test::FakeEncoder {
1020 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001021 explicit TestEncoder(TimeController* time_controller)
1022 : FakeEncoder(time_controller->GetClock()),
1023 time_controller_(time_controller) {
1024 RTC_DCHECK(time_controller_);
1025 }
perkj26091b12016-09-01 01:17:40 -07001026
Erik Språngaed30702018-11-05 12:57:17 +01001027 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001028 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001029 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001030 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001031 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001032 info.scaling_settings = VideoEncoder::ScalingSettings(
1033 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001034 }
1035 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001036 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1037 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001038 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001039 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001040 for (int tid = 0; tid < num_layers; ++tid)
1041 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001042 }
1043 }
Erik Språngaed30702018-11-05 12:57:17 +01001044 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001045
1046 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001047 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001048 info.apply_alignment_to_all_simulcast_layers =
1049 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001050 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001051 if (is_qp_trusted_.has_value()) {
1052 info.is_qp_trusted = is_qp_trusted_;
1053 }
Erik Språngaed30702018-11-05 12:57:17 +01001054 return info;
kthelgason876222f2016-11-29 01:44:11 -08001055 }
1056
Erik Språngb7cb7b52019-02-26 15:52:33 +01001057 int32_t RegisterEncodeCompleteCallback(
1058 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001059 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001060 encoded_image_callback_ = callback;
1061 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1062 }
1063
perkjfa10b552016-10-02 23:45:26 -07001064 void ContinueEncode() { continue_encode_event_.Set(); }
1065
1066 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1067 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001068 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001069 EXPECT_EQ(timestamp_, timestamp);
1070 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1071 }
1072
kthelgason2fc52542017-03-03 00:24:41 -08001073 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001074 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001075 quality_scaling_ = b;
1076 }
kthelgasonad9010c2017-02-14 00:46:51 -08001077
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001078 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001079 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001080 requested_resolution_alignment_ = requested_resolution_alignment;
1081 }
1082
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001083 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1084 MutexLock lock(&local_mutex_);
1085 apply_alignment_to_all_simulcast_layers_ = b;
1086 }
1087
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001088 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001089 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001090 is_hardware_accelerated_ = is_hardware_accelerated;
1091 }
1092
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001093 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1094 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001095 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001096 temporal_layers_supported_[spatial_idx] = supported;
1097 }
1098
Sergey Silkin6456e352019-07-08 17:56:40 +02001099 void SetResolutionBitrateLimits(
1100 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001101 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001102 resolution_bitrate_limits_ = thresholds;
1103 }
1104
sprangfe627f32017-03-29 08:24:59 -07001105 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001106 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001107 force_init_encode_failed_ = force_failure;
1108 }
1109
Niels Möller6bb5ab92019-01-11 11:11:10 +01001110 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001112 rate_factor_ = rate_factor;
1113 }
1114
Erik Språngd7329ca2019-02-21 21:19:53 +01001115 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001116 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001117 return last_framerate_;
1118 }
1119
Erik Språngd7329ca2019-02-21 21:19:53 +01001120 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001121 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001122 return last_update_rect_;
1123 }
1124
Niels Möller87e2d782019-03-07 10:18:23 +01001125 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001126 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001127 return last_frame_types_;
1128 }
1129
1130 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001131 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001132 keyframe ? VideoFrameType::kVideoFrameKey
1133 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001134 {
Markus Handella3765182020-07-08 13:13:32 +02001135 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001136 last_frame_types_ = frame_type;
1137 }
Niels Möllerb859b322019-03-07 12:40:01 +01001138 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001139 }
1140
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001141 void InjectEncodedImage(const EncodedImage& image,
1142 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001143 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001144 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001145 }
1146
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001147 void SetEncodedImageData(
1148 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001149 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001150 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001151 }
1152
Erik Språngd7329ca2019-02-21 21:19:53 +01001153 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001154 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001155 expect_null_frame_ = true;
1156 }
1157
Erik Språng5056af02019-09-02 15:53:11 +02001158 absl::optional<VideoEncoder::RateControlParameters>
1159 GetAndResetLastRateControlSettings() {
1160 auto settings = last_rate_control_settings_;
1161 last_rate_control_settings_.reset();
1162 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001163 }
1164
Henrik Boström56db9ff2021-03-24 09:06:45 +01001165 int GetLastInputWidth() const {
1166 MutexLock lock(&local_mutex_);
1167 return last_input_width_;
1168 }
1169
1170 int GetLastInputHeight() const {
1171 MutexLock lock(&local_mutex_);
1172 return last_input_height_;
1173 }
1174
Evan Shrubsole895556e2020-10-05 09:15:13 +02001175 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1176 MutexLock lock(&local_mutex_);
1177 return last_input_pixel_format_;
1178 }
1179
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001180 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001181 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001182 return num_set_rates_;
1183 }
1184
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001185 void SetPreferredPixelFormats(
1186 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1187 pixel_formats) {
1188 MutexLock lock(&local_mutex_);
1189 preferred_pixel_formats_ = std::move(pixel_formats);
1190 }
1191
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001192 void SetIsQpTrusted(absl::optional<bool> trusted) {
1193 MutexLock lock(&local_mutex_);
1194 is_qp_trusted_ = trusted;
1195 }
1196
perkjfa10b552016-10-02 23:45:26 -07001197 private:
perkj26091b12016-09-01 01:17:40 -07001198 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001199 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001200 {
Markus Handella3765182020-07-08 13:13:32 +02001201 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001202 if (expect_null_frame_) {
1203 EXPECT_EQ(input_image.timestamp(), 0u);
1204 EXPECT_EQ(input_image.width(), 1);
1205 last_frame_types_ = *frame_types;
1206 expect_null_frame_ = false;
1207 } else {
1208 EXPECT_GT(input_image.timestamp(), timestamp_);
1209 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1210 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1211 }
perkj26091b12016-09-01 01:17:40 -07001212
1213 timestamp_ = input_image.timestamp();
1214 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001215 last_input_width_ = input_image.width();
1216 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001217 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001218 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001219 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001220 }
Niels Möllerb859b322019-03-07 12:40:01 +01001221 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001222 return result;
1223 }
1224
Niels Möller08ae7ce2020-09-23 15:58:12 +02001225 CodecSpecificInfo EncodeHook(
1226 EncodedImage& encoded_image,
1227 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001228 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001229 {
1230 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001231 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001232 }
1233 MutexLock lock(&local_mutex_);
1234 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001235 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001236 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001237 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001238 }
1239
sprangfe627f32017-03-29 08:24:59 -07001240 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001241 const Settings& settings) override {
1242 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001243
Markus Handella3765182020-07-08 13:13:32 +02001244 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001245 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001246
Erik Språng82fad3d2018-03-21 09:57:23 +01001247 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001248 // Simulate setting up temporal layers, in order to validate the life
1249 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001250 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001251 frame_buffer_controller_ =
1252 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001253 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001254 if (force_init_encode_failed_) {
1255 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001256 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001257 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001258
Erik Språngb7cb7b52019-02-26 15:52:33 +01001259 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001260 return res;
1261 }
1262
Erik Språngb7cb7b52019-02-26 15:52:33 +01001263 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001264 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001265 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1266 initialized_ = EncoderState::kUninitialized;
1267 return FakeEncoder::Release();
1268 }
1269
Erik Språng16cb8f52019-04-12 13:59:09 +02001270 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001271 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001272 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001273 VideoBitrateAllocation adjusted_rate_allocation;
1274 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1275 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001276 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001277 adjusted_rate_allocation.SetBitrate(
1278 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001279 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001280 rate_factor_));
1281 }
1282 }
1283 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001284 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001285 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001286 RateControlParameters adjusted_paramters = parameters;
1287 adjusted_paramters.bitrate = adjusted_rate_allocation;
1288 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001289 }
1290
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001291 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001292 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001293 enum class EncoderState {
1294 kUninitialized,
1295 kInitializationFailed,
1296 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001297 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001298 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001299 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1300 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1301 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1302 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1303 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1304 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001305 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1306 false;
Markus Handella3765182020-07-08 13:13:32 +02001307 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001308 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1309 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001310 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001311 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001312 absl::optional<bool>
1313 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001314 local_mutex_);
1315 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1316 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1317 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001318 absl::optional<VideoEncoder::RateControlParameters>
1319 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001320 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1321 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001322 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001323 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001324 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1325 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001326 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001327 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001328 RTC_GUARDED_BY(local_mutex_);
1329 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001330 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1331 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001332 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1333 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001334 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001335 };
1336
mflodmancc3d4422017-08-03 08:27:51 -07001337 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001338 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001339 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1340 : time_controller_(time_controller), test_encoder_(test_encoder) {
1341 RTC_DCHECK(time_controller_);
1342 }
perkj26091b12016-09-01 01:17:40 -07001343
perkj26091b12016-09-01 01:17:40 -07001344 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001345 EXPECT_TRUE(
1346 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1347 }
1348
1349 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1350 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001351 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001352 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001353 return false;
perkj26091b12016-09-01 01:17:40 -07001354 {
Markus Handella3765182020-07-08 13:13:32 +02001355 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001356 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001357 }
1358 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001359 return true;
perkj26091b12016-09-01 01:17:40 -07001360 }
1361
sprangb1ca0732017-02-01 08:38:12 -08001362 void WaitForEncodedFrame(uint32_t expected_width,
1363 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001364 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001365 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001366 }
1367
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001368 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001369 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001370 uint32_t width = 0;
1371 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001372 {
Markus Handella3765182020-07-08 13:13:32 +02001373 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001374 width = last_width_;
1375 height = last_height_;
1376 }
1377 EXPECT_EQ(expected_height, height);
1378 EXPECT_EQ(expected_width, width);
1379 }
1380
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001381 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1382 VideoRotation rotation;
1383 {
Markus Handella3765182020-07-08 13:13:32 +02001384 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001385 rotation = last_rotation_;
1386 }
1387 EXPECT_EQ(expected_rotation, rotation);
1388 }
1389
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001390 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001391
sprangc5d62e22017-04-02 23:53:04 -07001392 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001393 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001394 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001395 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001396 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001397 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001398 }
1399
perkj26091b12016-09-01 01:17:40 -07001400 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001401 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001402 expect_frames_ = false;
1403 }
1404
asaperssonfab67072017-04-04 05:51:49 -07001405 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001406 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001407 return number_of_reconfigurations_;
1408 }
1409
asaperssonfab67072017-04-04 05:51:49 -07001410 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001411 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001412 return min_transmit_bitrate_bps_;
1413 }
1414
Erik Språngd7329ca2019-02-21 21:19:53 +01001415 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001416 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001417 num_expected_layers_ = num_layers;
1418 }
1419
Erik Språngb7cb7b52019-02-26 15:52:33 +01001420 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001421 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001422 return last_capture_time_ms_;
1423 }
1424
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001425 const EncodedImage& GetLastEncodedImage() {
1426 MutexLock lock(&mutex_);
1427 return last_encoded_image_;
1428 }
1429
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001430 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001431 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001432 return std::move(last_encoded_image_data_);
1433 }
1434
Per Kjellanderdcef6412020-10-07 15:09:05 +02001435 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1436 MutexLock lock(&mutex_);
1437 return last_bitrate_allocation_;
1438 }
1439
1440 int number_of_bitrate_allocations() const {
1441 MutexLock lock(&mutex_);
1442 return number_of_bitrate_allocations_;
1443 }
1444
Per Kjellandera9434842020-10-15 17:53:22 +02001445 VideoLayersAllocation GetLastVideoLayersAllocation() {
1446 MutexLock lock(&mutex_);
1447 return last_layers_allocation_;
1448 }
1449
1450 int number_of_layers_allocations() const {
1451 MutexLock lock(&mutex_);
1452 return number_of_layers_allocations_;
1453 }
1454
perkj26091b12016-09-01 01:17:40 -07001455 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001456 Result OnEncodedImage(
1457 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001458 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001459 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001460 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001461 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001462 last_encoded_image_data_ = std::vector<uint8_t>(
1463 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001464 uint32_t timestamp = encoded_image.Timestamp();
1465 if (last_timestamp_ != timestamp) {
1466 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001467 last_width_ = encoded_image._encodedWidth;
1468 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001469 } else {
1470 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001471 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1472 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001473 }
1474 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001475 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001476 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001477 if (num_received_layers_ == num_expected_layers_) {
1478 encoded_frame_event_.Set();
1479 }
sprangb1ca0732017-02-01 08:38:12 -08001480 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001481 }
1482
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001483 void OnEncoderConfigurationChanged(
1484 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001485 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001486 VideoEncoderConfig::ContentType content_type,
1487 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001488 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001489 ++number_of_reconfigurations_;
1490 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1491 }
1492
Per Kjellanderdcef6412020-10-07 15:09:05 +02001493 void OnBitrateAllocationUpdated(
1494 const VideoBitrateAllocation& allocation) override {
1495 MutexLock lock(&mutex_);
1496 ++number_of_bitrate_allocations_;
1497 last_bitrate_allocation_ = allocation;
1498 }
1499
Per Kjellandera9434842020-10-15 17:53:22 +02001500 void OnVideoLayersAllocationUpdated(
1501 VideoLayersAllocation allocation) override {
1502 MutexLock lock(&mutex_);
1503 ++number_of_layers_allocations_;
1504 last_layers_allocation_ = allocation;
1505 rtc::StringBuilder log;
1506 for (const auto& layer : allocation.active_spatial_layers) {
1507 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1508 << "[";
1509 for (const auto target_bitrate :
1510 layer.target_bitrate_per_temporal_layer) {
1511 log << target_bitrate.kbps() << ",";
1512 }
1513 log << "]";
1514 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001515 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001516 }
1517
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001518 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001519 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001520 TestEncoder* test_encoder_;
1521 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001522 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001523 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001524 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001525 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001526 uint32_t last_height_ = 0;
1527 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001528 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001529 size_t num_expected_layers_ = 1;
1530 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001531 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001532 int number_of_reconfigurations_ = 0;
1533 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001534 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1535 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001536 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1537 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001538 };
1539
Sergey Silkin5ee69672019-07-02 14:18:34 +02001540 class VideoBitrateAllocatorProxyFactory
1541 : public VideoBitrateAllocatorFactory {
1542 public:
1543 VideoBitrateAllocatorProxyFactory()
1544 : bitrate_allocator_factory_(
1545 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1546
1547 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1548 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001549 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001550 codec_config_ = codec;
1551 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1552 }
1553
1554 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001555 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001556 return codec_config_;
1557 }
1558
1559 private:
1560 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1561
Markus Handella3765182020-07-08 13:13:32 +02001562 mutable Mutex mutex_;
1563 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001564 };
1565
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001566 Clock* clock() { return time_controller_.GetClock(); }
1567 void AdvanceTime(TimeDelta duration) {
1568 time_controller_.AdvanceTime(duration);
1569 }
1570
1571 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1572
1573 protected:
1574 virtual TaskQueueFactory* GetTaskQueueFactory() {
1575 return time_controller_.GetTaskQueueFactory();
1576 }
1577
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001578 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001579 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001580 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001581 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001582 int codec_width_;
1583 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001584 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001585 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001586 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001587 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001588 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001589 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001590 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001591 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001592};
1593
mflodmancc3d4422017-08-03 08:27:51 -07001594TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001596 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001597 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001598 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001599 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001600 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001601 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001602}
1603
mflodmancc3d4422017-08-03 08:27:51 -07001604TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001605 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001606 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001607 // The encoder will cache up to one frame for a short duration. Adding two
1608 // frames means that the first frame will be dropped and the second frame will
1609 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001610 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001611 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001612 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001613 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001614 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001615
Henrik Boström381d1092020-05-12 18:49:07 +02001616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001617 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001618
Sebastian Janssona3177052018-04-10 13:05:49 +02001619 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001620 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001621 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1622
1623 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001624 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001625}
1626
mflodmancc3d4422017-08-03 08:27:51 -07001627TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001629 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001630 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001631 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001632
Henrik Boström381d1092020-05-12 18:49:07 +02001633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001634 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1635
Sebastian Janssona3177052018-04-10 13:05:49 +02001636 // The encoder will cache up to one frame for a short duration. Adding two
1637 // frames means that the first frame will be dropped and the second frame will
1638 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001639 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001640 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001641
Henrik Boström381d1092020-05-12 18:49:07 +02001642 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001643 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001644 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001645 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1646 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001647 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001648}
1649
mflodmancc3d4422017-08-03 08:27:51 -07001650TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001651 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001652 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001653 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001654 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001655
1656 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001657 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001658
perkja49cbd32016-09-16 07:53:41 -07001659 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001660 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001661 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001662}
1663
mflodmancc3d4422017-08-03 08:27:51 -07001664TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001666 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001667
perkja49cbd32016-09-16 07:53:41 -07001668 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001669 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001670
mflodmancc3d4422017-08-03 08:27:51 -07001671 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001672 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001673 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001674 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1675 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001676}
1677
Markus Handell9a478b52021-11-18 16:07:01 +01001678TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1679 test::FrameForwarder source;
1680 video_stream_encoder_->SetSource(&source,
1681 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001683 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001684
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001685 int dropped_count = 0;
1686 stats_proxy_->SetDroppedFrameCallback(
1687 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1688 ++dropped_count;
1689 });
1690
Markus Handell9a478b52021-11-18 16:07:01 +01001691 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1692 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1693 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001694 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001695 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001696}
1697
Henrik Boström56db9ff2021-03-24 09:06:45 +01001698TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001700 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001701
1702 rtc::Event frame_destroyed_event;
1703 video_source_.IncomingCapturedFrame(
1704 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001705 WaitForEncodedFrame(1);
1706 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1707 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001708 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1709 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001710 video_stream_encoder_->Stop();
1711}
1712
Henrik Boström56db9ff2021-03-24 09:06:45 +01001713TEST_F(VideoStreamEncoderTest,
1714 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001715 // Use the cropping factory.
1716 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001717 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001718 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1719 kMaxPayloadLength);
1720 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1721
1722 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001725 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1726 WaitForEncodedFrame(1);
1727 // The encoder will have been configured once.
1728 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001729 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1730 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001731
1732 // Now send in a fake frame that needs to be cropped as the width/height
1733 // aren't divisible by 4 (see CreateEncoderStreams above).
1734 rtc::Event frame_destroyed_event;
1735 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1736 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001737 WaitForEncodedFrame(2);
1738 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1739 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001740 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1741 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001742 video_stream_encoder_->Stop();
1743}
1744
Evan Shrubsole895556e2020-10-05 09:15:13 +02001745TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1746 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001747 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001748
1749 video_source_.IncomingCapturedFrame(
1750 CreateNV12Frame(1, codec_width_, codec_height_));
1751 WaitForEncodedFrame(1);
1752 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1753 fake_encoder_.GetLastInputPixelFormat());
1754 video_stream_encoder_->Stop();
1755}
1756
Henrik Boström56db9ff2021-03-24 09:06:45 +01001757TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001759 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001760
1761 fake_encoder_.SetPreferredPixelFormats({});
1762
1763 rtc::Event frame_destroyed_event;
1764 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1765 1, &frame_destroyed_event, codec_width_, codec_height_));
1766 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001767 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001768 fake_encoder_.GetLastInputPixelFormat());
1769 video_stream_encoder_->Stop();
1770}
1771
Henrik Boström56db9ff2021-03-24 09:06:45 +01001772TEST_F(VideoStreamEncoderTest,
1773 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001775 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001776
1777 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1778
1779 rtc::Event frame_destroyed_event;
1780 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1781 1, &frame_destroyed_event, codec_width_, codec_height_));
1782 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001783 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001784 fake_encoder_.GetLastInputPixelFormat());
1785 video_stream_encoder_->Stop();
1786}
1787
Henrik Boström56db9ff2021-03-24 09:06:45 +01001788TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001790 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001791
1792 // Fake NV12 native frame does not allow mapping to I444.
1793 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1794
1795 rtc::Event frame_destroyed_event;
1796 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1797 1, &frame_destroyed_event, codec_width_, codec_height_));
1798 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001799 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001800 fake_encoder_.GetLastInputPixelFormat());
1801 video_stream_encoder_->Stop();
1802}
1803
Henrik Boström56db9ff2021-03-24 09:06:45 +01001804TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001805 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001806 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001807
1808 rtc::Event frame_destroyed_event;
1809 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1810 1, &frame_destroyed_event, codec_width_, codec_height_));
1811 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001812 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001813 fake_encoder_.GetLastInputPixelFormat());
1814 video_stream_encoder_->Stop();
1815}
1816
Ying Wang9b881ab2020-02-07 14:29:32 +01001817TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001818 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001819 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001820 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1821 WaitForEncodedFrame(1);
1822
Henrik Boström381d1092020-05-12 18:49:07 +02001823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001825 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1826 // frames. Adding two frames means that the first frame will be dropped and
1827 // the second frame will be sent to the encoder.
1828 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1829 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1830 WaitForEncodedFrame(3);
1831 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1832 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1833 WaitForEncodedFrame(5);
1834 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1835 video_stream_encoder_->Stop();
1836}
1837
mflodmancc3d4422017-08-03 08:27:51 -07001838TEST_F(VideoStreamEncoderTest,
1839 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001840 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001841 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001842 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001843
1844 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001845 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001846 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001847 // The encoder will have been configured once when the first frame is
1848 // received.
1849 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001850
1851 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001852 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001853 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001854 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001855 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001856
1857 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001858 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001859 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001860 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001861 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001862
mflodmancc3d4422017-08-03 08:27:51 -07001863 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001864}
1865
mflodmancc3d4422017-08-03 08:27:51 -07001866TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001867 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001868 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001869
1870 // Capture a frame and wait for it to synchronize with the encoder thread.
1871 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001872 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001873 // The encoder will have been configured once.
1874 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001875 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1876 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001877
1878 codec_width_ *= 2;
1879 codec_height_ *= 2;
1880 // Capture a frame with a higher resolution and wait for it to synchronize
1881 // with the encoder thread.
1882 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001883 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001884 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1885 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001886 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001887
mflodmancc3d4422017-08-03 08:27:51 -07001888 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001889}
1890
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001891TEST_F(VideoStreamEncoderTest,
1892 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001894 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001895
1896 // Capture a frame and wait for it to synchronize with the encoder thread.
1897 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1898 WaitForEncodedFrame(1);
1899
1900 VideoEncoderConfig video_encoder_config;
1901 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1902 // Changing the max payload data length recreates encoder.
1903 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1904 kMaxPayloadLength / 2);
1905
1906 // Capture a frame and wait for it to synchronize with the encoder thread.
1907 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1908 WaitForEncodedFrame(2);
1909 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1910
1911 video_stream_encoder_->Stop();
1912}
1913
Sergey Silkin5ee69672019-07-02 14:18:34 +02001914TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001915 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001916 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001917
1918 VideoEncoderConfig video_encoder_config;
1919 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001920 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1921 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001922 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1923 kMaxPayloadLength);
1924
1925 // Capture a frame and wait for it to synchronize with the encoder thread.
1926 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1927 WaitForEncodedFrame(1);
1928 // The encoder will have been configured once when the first frame is
1929 // received.
1930 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001931 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001932 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001933 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001934 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1935
Sergey Silkin6456e352019-07-08 17:56:40 +02001936 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1937 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001938 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1939 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001940 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1941 kMaxPayloadLength);
1942
1943 // Capture a frame and wait for it to synchronize with the encoder thread.
1944 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1945 WaitForEncodedFrame(2);
1946 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1947 // Bitrate limits have changed - rate allocator should be reconfigured,
1948 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001949 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001950 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001951 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001952 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001953 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001954
1955 video_stream_encoder_->Stop();
1956}
1957
Sergey Silkin6456e352019-07-08 17:56:40 +02001958TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001959 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001961 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001962
Sergey Silkincd02eba2020-01-20 14:48:40 +01001963 const uint32_t kMinEncBitrateKbps = 100;
1964 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001965 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001966 /*frame_size_pixels=*/codec_width_ * codec_height_,
1967 /*min_start_bitrate_bps=*/0,
1968 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1969 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001970 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1971
Sergey Silkincd02eba2020-01-20 14:48:40 +01001972 VideoEncoderConfig video_encoder_config;
1973 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1974 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1975 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1976 (kMinEncBitrateKbps + 1) * 1000;
1977 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1978 kMaxPayloadLength);
1979
1980 // When both encoder and app provide bitrate limits, the intersection of
1981 // provided sets should be used.
1982 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1983 WaitForEncodedFrame(1);
1984 EXPECT_EQ(kMaxEncBitrateKbps,
1985 bitrate_allocator_factory_.codec_config().maxBitrate);
1986 EXPECT_EQ(kMinEncBitrateKbps + 1,
1987 bitrate_allocator_factory_.codec_config().minBitrate);
1988
1989 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1990 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1991 (kMinEncBitrateKbps - 1) * 1000;
1992 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1993 kMaxPayloadLength);
1994 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001996 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001997 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001998 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001999 bitrate_allocator_factory_.codec_config().minBitrate);
2000
Sergey Silkincd02eba2020-01-20 14:48:40 +01002001 video_stream_encoder_->Stop();
2002}
2003
2004TEST_F(VideoStreamEncoderTest,
2005 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002007 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002008
2009 const uint32_t kMinAppBitrateKbps = 100;
2010 const uint32_t kMaxAppBitrateKbps = 200;
2011 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2012 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2013 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2014 /*frame_size_pixels=*/codec_width_ * codec_height_,
2015 /*min_start_bitrate_bps=*/0,
2016 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2017 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2018 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2019
2020 VideoEncoderConfig video_encoder_config;
2021 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2022 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2023 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2024 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2026 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002027
Sergey Silkincd02eba2020-01-20 14:48:40 +01002028 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2029 WaitForEncodedFrame(1);
2030 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002031 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002032 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002033 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002034
2035 video_stream_encoder_->Stop();
2036}
2037
2038TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002039 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002041 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002042
2043 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002044 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002045 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002046 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002047 fake_encoder_.SetResolutionBitrateLimits(
2048 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2049
2050 VideoEncoderConfig video_encoder_config;
2051 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2052 video_encoder_config.max_bitrate_bps = 0;
2053 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2054 kMaxPayloadLength);
2055
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002056 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002057 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2058 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2060 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002061 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2062 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2063
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002065 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2066 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002067 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2068 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002069 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2070 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2071
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002072 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002073 // encoder for 360p should be used.
2074 video_source_.IncomingCapturedFrame(
2075 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2076 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002077 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2078 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002079 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2080 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2081
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002082 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002083 // ignored.
2084 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2085 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002086 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2087 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002088 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2089 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002090 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2091 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002092 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2093 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2094
2095 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2096 // for 270p should be used.
2097 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2098 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002099 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2100 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002101 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2102 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2103
2104 video_stream_encoder_->Stop();
2105}
2106
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002107TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002109 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002110
2111 VideoEncoderConfig video_encoder_config;
2112 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2113 video_encoder_config.max_bitrate_bps = 0;
2114 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2115 kMaxPayloadLength);
2116
2117 // Encode 720p frame to get the default encoder target bitrate.
2118 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2119 WaitForEncodedFrame(1);
2120 const uint32_t kDefaultTargetBitrateFor720pKbps =
2121 bitrate_allocator_factory_.codec_config()
2122 .simulcastStream[0]
2123 .targetBitrate;
2124
2125 // Set the max recommended encoder bitrate to something lower than the default
2126 // target bitrate.
2127 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2128 1280 * 720, 10 * 1000, 10 * 1000,
2129 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2130 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2131
2132 // Change resolution to trigger encoder reinitialization.
2133 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2134 WaitForEncodedFrame(2);
2135 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2136 WaitForEncodedFrame(3);
2137
2138 // Ensure the target bitrate is capped by the max bitrate.
2139 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2140 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2141 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2142 .simulcastStream[0]
2143 .targetBitrate *
2144 1000,
2145 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2146
2147 video_stream_encoder_->Stop();
2148}
2149
Åsa Perssona7e34d32021-01-20 15:36:13 +01002150TEST_F(VideoStreamEncoderTest,
2151 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2152 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2153 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2154 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2155 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2156 fake_encoder_.SetResolutionBitrateLimits(
2157 {kEncoderLimits270p, kEncoderLimits360p});
2158
2159 // Two streams, highest stream active.
2160 VideoEncoderConfig config;
2161 const int kNumStreams = 2;
2162 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2163 config.max_bitrate_bps = 0;
2164 config.simulcast_layers[0].active = false;
2165 config.simulcast_layers[1].active = true;
2166 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002167 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002168 "VP8", /*max qp*/ 56, /*screencast*/ false,
2169 /*screenshare enabled*/ false);
2170 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2171
2172 // The encoder bitrate limits for 270p should be used.
2173 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2174 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002175 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002176 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002177 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002178 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002179 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002180
2181 // The encoder bitrate limits for 360p should be used.
2182 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2183 EXPECT_FALSE(WaitForFrame(1000));
2184 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002185 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002186 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002187 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002188
2189 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2190 video_source_.IncomingCapturedFrame(
2191 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2192 EXPECT_FALSE(WaitForFrame(1000));
2193 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002194 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002195 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002196 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002197
2198 // Resolution higher than 360p. Encoder limits should be ignored.
2199 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2200 EXPECT_FALSE(WaitForFrame(1000));
2201 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002202 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002203 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002204 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002205 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002206 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002207 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002208 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002209
2210 // Resolution lower than 270p. The encoder limits for 270p should be used.
2211 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2212 EXPECT_FALSE(WaitForFrame(1000));
2213 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002214 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002215 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002216 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002217
2218 video_stream_encoder_->Stop();
2219}
2220
2221TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002222 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2223 // Two streams, highest stream active.
2224 VideoEncoderConfig config;
2225 const int kNumStreams = 2;
2226 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2227 config.max_bitrate_bps = 0;
2228 config.simulcast_layers[0].active = false;
2229 config.simulcast_layers[1].active = true;
2230 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002231 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002232 "VP8", /*max qp*/ 56, /*screencast*/ false,
2233 /*screenshare enabled*/ false);
2234 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2235
2236 // Default bitrate limits for 270p should be used.
2237 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2238 kDefaultLimits270p =
2239 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002240 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002241 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2242 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002243 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002244 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002245 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002246 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002247 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002248
2249 // Default bitrate limits for 360p should be used.
2250 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2251 kDefaultLimits360p =
2252 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002253 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002254 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2255 EXPECT_FALSE(WaitForFrame(1000));
2256 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002257 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002258 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002259 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002260
2261 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2262 video_source_.IncomingCapturedFrame(
2263 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2264 EXPECT_FALSE(WaitForFrame(1000));
2265 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002266 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002267 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002268 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002269
2270 // Default bitrate limits for 540p should be used.
2271 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2272 kDefaultLimits540p =
2273 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002274 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002275 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2276 EXPECT_FALSE(WaitForFrame(1000));
2277 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002278 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002279 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002280 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002281
2282 video_stream_encoder_->Stop();
2283}
2284
2285TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002286 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2287 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2288 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2289 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2290 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2291 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2292 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2293 fake_encoder_.SetResolutionBitrateLimits(
2294 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2295
2296 // Three streams, middle stream active.
2297 VideoEncoderConfig config;
2298 const int kNumStreams = 3;
2299 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2300 config.simulcast_layers[0].active = false;
2301 config.simulcast_layers[1].active = true;
2302 config.simulcast_layers[2].active = false;
2303 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002304 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002305 "VP8", /*max qp*/ 56, /*screencast*/ false,
2306 /*screenshare enabled*/ false);
2307 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2308
2309 // The encoder bitrate limits for 360p should be used.
2310 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2311 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002312 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002313 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002314 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002315 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002316 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002317
2318 // The encoder bitrate limits for 270p should be used.
2319 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2320 EXPECT_FALSE(WaitForFrame(1000));
2321 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002322 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002323 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002324 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002325
2326 video_stream_encoder_->Stop();
2327}
2328
2329TEST_F(VideoStreamEncoderTest,
2330 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2331 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2332 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2333 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2334 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2335 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2336 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2337 fake_encoder_.SetResolutionBitrateLimits(
2338 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2339
2340 // Three streams, lowest stream active.
2341 VideoEncoderConfig config;
2342 const int kNumStreams = 3;
2343 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2344 config.simulcast_layers[0].active = true;
2345 config.simulcast_layers[1].active = false;
2346 config.simulcast_layers[2].active = false;
2347 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002348 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002349 "VP8", /*max qp*/ 56, /*screencast*/ false,
2350 /*screenshare enabled*/ false);
2351 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2352
2353 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2354 // on lowest stream, limits for 270p should not be used
2355 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2356 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002357 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002358 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002359 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002360 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002361 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002362
2363 video_stream_encoder_->Stop();
2364}
2365
2366TEST_F(VideoStreamEncoderTest,
2367 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2368 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2369 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2370 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2371 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2372 fake_encoder_.SetResolutionBitrateLimits(
2373 {kEncoderLimits270p, kEncoderLimits360p});
2374 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2375
2376 // Two streams, highest stream active.
2377 VideoEncoderConfig config;
2378 const int kNumStreams = 2;
2379 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2380 config.simulcast_layers[0].active = false;
2381 config.simulcast_layers[1].active = true;
2382 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2383 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002384 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002385 "VP8", /*max qp*/ 56, /*screencast*/ false,
2386 /*screenshare enabled*/ false);
2387 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2388
2389 // The encoder bitrate limits for 270p should be used.
2390 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2391 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002392 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002393 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002394 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002395 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002396 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002397
2398 // The max configured bitrate is less than the encoder limit for 360p.
2399 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2400 EXPECT_FALSE(WaitForFrame(1000));
2401 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002402 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002403 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002404 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002405
2406 video_stream_encoder_->Stop();
2407}
2408
mflodmancc3d4422017-08-03 08:27:51 -07002409TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002410 EXPECT_TRUE(video_source_.has_sinks());
2411 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002412 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002413 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002414 EXPECT_FALSE(video_source_.has_sinks());
2415 EXPECT_TRUE(new_video_source.has_sinks());
2416
mflodmancc3d4422017-08-03 08:27:51 -07002417 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002418}
2419
mflodmancc3d4422017-08-03 08:27:51 -07002420TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002421 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002422 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002423 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002424 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002425}
2426
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002427class ResolutionAlignmentTest
2428 : public VideoStreamEncoderTest,
2429 public ::testing::WithParamInterface<
2430 ::testing::tuple<int, std::vector<double>>> {
2431 public:
2432 ResolutionAlignmentTest()
2433 : requested_alignment_(::testing::get<0>(GetParam())),
2434 scale_factors_(::testing::get<1>(GetParam())) {}
2435
2436 protected:
2437 const int requested_alignment_;
2438 const std::vector<double> scale_factors_;
2439};
2440
2441INSTANTIATE_TEST_SUITE_P(
2442 AlignmentAndScaleFactors,
2443 ResolutionAlignmentTest,
2444 ::testing::Combine(
2445 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2446 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2447 std::vector<double>{-1.0, -1.0},
2448 std::vector<double>{-1.0, -1.0, -1.0},
2449 std::vector<double>{4.0, 2.0, 1.0},
2450 std::vector<double>{9999.0, -1.0, 1.0},
2451 std::vector<double>{3.99, 2.01, 1.0},
2452 std::vector<double>{4.9, 1.7, 1.25},
2453 std::vector<double>{10.0, 4.0, 3.0},
2454 std::vector<double>{1.75, 3.5},
2455 std::vector<double>{1.5, 2.5},
2456 std::vector<double>{1.3, 1.0})));
2457
2458TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2459 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002460 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002461 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2462 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2463
2464 // Fill config with the scaling factor by which to reduce encoding size.
2465 const int num_streams = scale_factors_.size();
2466 VideoEncoderConfig config;
2467 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2468 for (int i = 0; i < num_streams; ++i) {
2469 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2470 }
2471 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002472 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002473 "VP8", /*max qp*/ 56, /*screencast*/ false,
2474 /*screenshare enabled*/ false);
2475 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2476
Henrik Boström381d1092020-05-12 18:49:07 +02002477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002478 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2479 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002480 // Wait for all layers before triggering event.
2481 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002482
2483 // On the 1st frame, we should have initialized the encoder and
2484 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002485 int64_t timestamp_ms = kFrameIntervalMs;
2486 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2487 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002488 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002489
2490 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2491 // (It's up the to the encoder to potentially drop the previous frame,
2492 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002493 timestamp_ms += kFrameIntervalMs;
2494 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2495 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002496 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002497
Asa Persson606d3cb2021-10-04 10:07:11 +02002498 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002499 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2500 // Frame size should be a multiple of the requested alignment.
2501 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2502 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2503 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2504 // Aspect ratio should match.
2505 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2506 codec.height * codec.simulcastStream[i].width);
2507 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002508
2509 video_stream_encoder_->Stop();
2510}
2511
Jonathan Yubc771b72017-12-08 17:04:29 -08002512TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2513 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002514 const int kWidth = 1280;
2515 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002516
2517 // We rely on the automatic resolution adaptation, but we handle framerate
2518 // adaptation manually by mocking the stats proxy.
2519 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002520
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002521 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002523 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002524 video_stream_encoder_->SetSource(&video_source_,
2525 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002526 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002527 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002528 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002529 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2530
Jonathan Yubc771b72017-12-08 17:04:29 -08002531 // Adapt down as far as possible.
2532 rtc::VideoSinkWants last_wants;
2533 int64_t t = 1;
2534 int loop_count = 0;
2535 do {
2536 ++loop_count;
2537 last_wants = video_source_.sink_wants();
2538
2539 // Simulate the framerate we've been asked to adapt to.
2540 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2541 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2542 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2543 mock_stats.input_frame_rate = fps;
2544 stats_proxy_->SetMockStats(mock_stats);
2545
2546 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2547 sink_.WaitForEncodedFrame(t);
2548 t += frame_interval_ms;
2549
mflodmancc3d4422017-08-03 08:27:51 -07002550 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002551 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002552 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002553 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2554 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002555 } while (video_source_.sink_wants().max_pixel_count <
2556 last_wants.max_pixel_count ||
2557 video_source_.sink_wants().max_framerate_fps <
2558 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002559
Jonathan Yubc771b72017-12-08 17:04:29 -08002560 // Verify that we've adapted all the way down.
2561 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002562 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002563 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2564 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002565 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002566 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2567 *video_source_.last_sent_height());
2568 EXPECT_EQ(kMinBalancedFramerateFps,
2569 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002570
Jonathan Yubc771b72017-12-08 17:04:29 -08002571 // Adapt back up the same number of times we adapted down.
2572 for (int i = 0; i < loop_count - 1; ++i) {
2573 last_wants = video_source_.sink_wants();
2574
2575 // Simulate the framerate we've been asked to adapt to.
2576 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2577 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2578 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2579 mock_stats.input_frame_rate = fps;
2580 stats_proxy_->SetMockStats(mock_stats);
2581
2582 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2583 sink_.WaitForEncodedFrame(t);
2584 t += frame_interval_ms;
2585
Henrik Boström91aa7322020-04-28 12:24:33 +02002586 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002587 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002588 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002589 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2590 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2592 last_wants.max_pixel_count ||
2593 video_source_.sink_wants().max_framerate_fps >
2594 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002595 }
2596
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002597 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002598 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002600 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2601 EXPECT_EQ((loop_count - 1) * 2,
2602 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002603
mflodmancc3d4422017-08-03 08:27:51 -07002604 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002605}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002606
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002607TEST_F(VideoStreamEncoderTest,
2608 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002609 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2610 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002611 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002612
2613 const int kFrameWidth = 1280;
2614 const int kFrameHeight = 720;
2615
2616 int64_t ntp_time = kFrameIntervalMs;
2617
2618 // Force an input frame rate to be available, or the adaptation call won't
2619 // know what framerate to adapt form.
2620 const int kInputFps = 30;
2621 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2622 stats.input_frame_rate = kInputFps;
2623 stats_proxy_->SetMockStats(stats);
2624
2625 video_source_.set_adaptation_enabled(true);
2626 video_stream_encoder_->SetSource(
2627 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002628 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002629 video_source_.IncomingCapturedFrame(
2630 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2631 sink_.WaitForEncodedFrame(ntp_time);
2632 ntp_time += kFrameIntervalMs;
2633
2634 // Trigger CPU overuse.
2635 video_stream_encoder_->TriggerCpuOveruse();
2636 video_source_.IncomingCapturedFrame(
2637 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2638 sink_.WaitForEncodedFrame(ntp_time);
2639 ntp_time += kFrameIntervalMs;
2640
2641 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2642 EXPECT_EQ(std::numeric_limits<int>::max(),
2643 video_source_.sink_wants().max_pixel_count);
2644 // Some framerate constraint should be set.
2645 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2646 EXPECT_LT(restricted_fps, kInputFps);
2647 video_source_.IncomingCapturedFrame(
2648 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2649 sink_.WaitForEncodedFrame(ntp_time);
2650 ntp_time += 100;
2651
Henrik Boström2671dac2020-05-19 16:29:09 +02002652 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002653 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2654 // Give the encoder queue time to process the change in degradation preference
2655 // by waiting for an encoded frame.
2656 video_source_.IncomingCapturedFrame(
2657 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2658 sink_.WaitForEncodedFrame(ntp_time);
2659 ntp_time += kFrameIntervalMs;
2660
2661 video_stream_encoder_->TriggerQualityLow();
2662 video_source_.IncomingCapturedFrame(
2663 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2664 sink_.WaitForEncodedFrame(ntp_time);
2665 ntp_time += kFrameIntervalMs;
2666
2667 // Some resolution constraint should be set.
2668 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2669 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2670 kFrameWidth * kFrameHeight);
2671 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2672
2673 int pixel_count = video_source_.sink_wants().max_pixel_count;
2674 // Triggering a CPU underuse should not change the sink wants since it has
2675 // not been overused for resolution since we changed degradation preference.
2676 video_stream_encoder_->TriggerCpuUnderuse();
2677 video_source_.IncomingCapturedFrame(
2678 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2679 sink_.WaitForEncodedFrame(ntp_time);
2680 ntp_time += kFrameIntervalMs;
2681 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2682 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2683
Evan Shrubsole64469032020-06-11 10:45:29 +02002684 // Change the degradation preference back. CPU underuse should not adapt since
2685 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002686 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002687 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2688 video_source_.IncomingCapturedFrame(
2689 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2690 sink_.WaitForEncodedFrame(ntp_time);
2691 ntp_time += 100;
2692 // Resolution adaptations is gone after changing degradation preference.
2693 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2694 EXPECT_EQ(std::numeric_limits<int>::max(),
2695 video_source_.sink_wants().max_pixel_count);
2696 // The fps adaptation from above is now back.
2697 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2698
2699 // Trigger CPU underuse.
2700 video_stream_encoder_->TriggerCpuUnderuse();
2701 video_source_.IncomingCapturedFrame(
2702 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2703 sink_.WaitForEncodedFrame(ntp_time);
2704 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002705 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2706
2707 // Trigger QP underuse, fps should return to normal.
2708 video_stream_encoder_->TriggerQualityHigh();
2709 video_source_.IncomingCapturedFrame(
2710 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2711 sink_.WaitForEncodedFrame(ntp_time);
2712 ntp_time += kFrameIntervalMs;
2713 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002714
2715 video_stream_encoder_->Stop();
2716}
2717
mflodmancc3d4422017-08-03 08:27:51 -07002718TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002719 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002720 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002721 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002722
sprangc5d62e22017-04-02 23:53:04 -07002723 const int kFrameWidth = 1280;
2724 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002725
Åsa Persson8c1bf952018-09-13 10:42:19 +02002726 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002727
kthelgason5e13d412016-12-01 03:59:51 -08002728 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002729 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002730 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002731 frame_timestamp += kFrameIntervalMs;
2732
perkj803d97f2016-11-01 11:45:46 -07002733 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002734 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002735 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002736 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002737 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002738 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002739
asapersson0944a802017-04-07 00:57:58 -07002740 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002741 // wanted resolution.
2742 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2743 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2744 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002745 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002746
2747 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002748 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002749 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002750 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002751 // Give the encoder queue time to process the change in degradation preference
2752 // by waiting for an encoded frame.
2753 new_video_source.IncomingCapturedFrame(
2754 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2755 sink_.WaitForEncodedFrame(frame_timestamp);
2756 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002757 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002758 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002759
sprangc5d62e22017-04-02 23:53:04 -07002760 // Force an input frame rate to be available, or the adaptation call won't
2761 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002762 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002763 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002764 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002765 stats_proxy_->SetMockStats(stats);
2766
mflodmancc3d4422017-08-03 08:27:51 -07002767 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002768 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002769 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002771 frame_timestamp += kFrameIntervalMs;
2772
2773 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002774 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002775 EXPECT_EQ(std::numeric_limits<int>::max(),
2776 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002777 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002778
asapersson02465b82017-04-10 01:12:52 -07002779 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002780 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2781 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002782 // Give the encoder queue time to process the change in degradation preference
2783 // by waiting for an encoded frame.
2784 new_video_source.IncomingCapturedFrame(
2785 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2786 sink_.WaitForEncodedFrame(frame_timestamp);
2787 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002788 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002789
mflodmancc3d4422017-08-03 08:27:51 -07002790 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002791 new_video_source.IncomingCapturedFrame(
2792 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002793 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002794 frame_timestamp += kFrameIntervalMs;
2795
2796 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002797 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002798
2799 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002800 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002801 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002802 // Give the encoder queue time to process the change in degradation preference
2803 // by waiting for an encoded frame.
2804 new_video_source.IncomingCapturedFrame(
2805 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2806 sink_.WaitForEncodedFrame(frame_timestamp);
2807 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002808 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2809 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002810 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002811 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002812
2813 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002814 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002815 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002816 // Give the encoder queue time to process the change in degradation preference
2817 // by waiting for an encoded frame.
2818 new_video_source.IncomingCapturedFrame(
2819 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2820 sink_.WaitForEncodedFrame(frame_timestamp);
2821 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002822 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2823 EXPECT_EQ(std::numeric_limits<int>::max(),
2824 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002825 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002826
mflodmancc3d4422017-08-03 08:27:51 -07002827 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002828}
2829
mflodmancc3d4422017-08-03 08:27:51 -07002830TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002833
asaperssonfab67072017-04-04 05:51:49 -07002834 const int kWidth = 1280;
2835 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002836 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002837 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002838 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2839 EXPECT_FALSE(stats.bw_limited_resolution);
2840 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2841
2842 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002843 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002844 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002845 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002846
2847 stats = stats_proxy_->GetStats();
2848 EXPECT_TRUE(stats.bw_limited_resolution);
2849 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2850
2851 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002852 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002853 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002854 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002855
2856 stats = stats_proxy_->GetStats();
2857 EXPECT_FALSE(stats.bw_limited_resolution);
2858 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2859 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2860
mflodmancc3d4422017-08-03 08:27:51 -07002861 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002862}
2863
mflodmancc3d4422017-08-03 08:27:51 -07002864TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002866 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002867
2868 const int kWidth = 1280;
2869 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002870 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002871 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002872 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2873 EXPECT_FALSE(stats.cpu_limited_resolution);
2874 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2875
2876 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002878 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002879 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002880
2881 stats = stats_proxy_->GetStats();
2882 EXPECT_TRUE(stats.cpu_limited_resolution);
2883 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2884
2885 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002886 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002887 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002888 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002889
2890 stats = stats_proxy_->GetStats();
2891 EXPECT_FALSE(stats.cpu_limited_resolution);
2892 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002893 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002894
mflodmancc3d4422017-08-03 08:27:51 -07002895 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002896}
2897
mflodmancc3d4422017-08-03 08:27:51 -07002898TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002899 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002900 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002901
asaperssonfab67072017-04-04 05:51:49 -07002902 const int kWidth = 1280;
2903 const int kHeight = 720;
2904 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002905 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002906 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002907 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002908 EXPECT_FALSE(stats.cpu_limited_resolution);
2909 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2910
asaperssonfab67072017-04-04 05:51:49 -07002911 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002912 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002913 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002914 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002915 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002916 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002917 EXPECT_TRUE(stats.cpu_limited_resolution);
2918 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2919
2920 // Set new source with adaptation still enabled.
2921 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002922 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002923 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002924
asaperssonfab67072017-04-04 05:51:49 -07002925 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002926 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002927 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002928 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002929 EXPECT_TRUE(stats.cpu_limited_resolution);
2930 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2931
2932 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002933 video_stream_encoder_->SetSource(&new_video_source,
2934 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002935
asaperssonfab67072017-04-04 05:51:49 -07002936 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002937 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002938 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002939 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002940 EXPECT_FALSE(stats.cpu_limited_resolution);
2941 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2942
2943 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002944 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002945 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002946
asaperssonfab67072017-04-04 05:51:49 -07002947 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002948 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002949 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002950 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002951 EXPECT_TRUE(stats.cpu_limited_resolution);
2952 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2953
asaperssonfab67072017-04-04 05:51:49 -07002954 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002955 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002956 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002957 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002958 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002959 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002960 EXPECT_FALSE(stats.cpu_limited_resolution);
2961 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002962 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002963
mflodmancc3d4422017-08-03 08:27:51 -07002964 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002965}
2966
mflodmancc3d4422017-08-03 08:27:51 -07002967TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002969 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002970
asaperssonfab67072017-04-04 05:51:49 -07002971 const int kWidth = 1280;
2972 const int kHeight = 720;
2973 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002974 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002975 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002976 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002977 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002978 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002979
2980 // Set new source with adaptation still enabled.
2981 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002982 video_stream_encoder_->SetSource(&new_video_source,
2983 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002984
asaperssonfab67072017-04-04 05:51:49 -07002985 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002986 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002987 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002988 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002989 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002990 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002991
asaperssonfab67072017-04-04 05:51:49 -07002992 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002993 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002994 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002995 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002996 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002997 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002998 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002999 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003000
asaperssonfab67072017-04-04 05:51:49 -07003001 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003002 video_stream_encoder_->SetSource(&new_video_source,
3003 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003004
asaperssonfab67072017-04-04 05:51:49 -07003005 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003006 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003007 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003008 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003009 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003010 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003011
asapersson02465b82017-04-10 01:12:52 -07003012 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003013 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003014 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003015
asaperssonfab67072017-04-04 05:51:49 -07003016 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003017 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003018 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003019 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003020 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003021 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3022 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003023
mflodmancc3d4422017-08-03 08:27:51 -07003024 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003025}
3026
mflodmancc3d4422017-08-03 08:27:51 -07003027TEST_F(VideoStreamEncoderTest,
3028 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003030 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003031
3032 const int kWidth = 1280;
3033 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003034 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003035 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003036 video_source_.IncomingCapturedFrame(
3037 CreateFrame(timestamp_ms, kWidth, kHeight));
3038 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003039 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3040 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3041 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3042
3043 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003044 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003045 timestamp_ms += kFrameIntervalMs;
3046 video_source_.IncomingCapturedFrame(
3047 CreateFrame(timestamp_ms, kWidth, kHeight));
3048 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003049 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3050 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3051 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3052
3053 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003054 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003055 timestamp_ms += kFrameIntervalMs;
3056 video_source_.IncomingCapturedFrame(
3057 CreateFrame(timestamp_ms, kWidth, kHeight));
3058 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003059 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3060 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3061 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3062
Niels Möller4db138e2018-04-19 09:04:13 +02003063 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003064 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003065
3066 VideoEncoderConfig video_encoder_config;
3067 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3068 // Make format different, to force recreation of encoder.
3069 video_encoder_config.video_format.parameters["foo"] = "foo";
3070 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003071 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003072 timestamp_ms += kFrameIntervalMs;
3073 video_source_.IncomingCapturedFrame(
3074 CreateFrame(timestamp_ms, kWidth, kHeight));
3075 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003076 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3077 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3078 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3079
mflodmancc3d4422017-08-03 08:27:51 -07003080 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003081}
3082
mflodmancc3d4422017-08-03 08:27:51 -07003083TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003084 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003086 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003087
3088 const int kWidth = 1280;
3089 const int kHeight = 720;
3090 int sequence = 1;
3091
3092 // Enable BALANCED preference, no initial limitation.
3093 test::FrameForwarder source;
3094 video_stream_encoder_->SetSource(&source,
3095 webrtc::DegradationPreference::BALANCED);
3096 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3097 WaitForEncodedFrame(sequence++);
3098 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3099 EXPECT_FALSE(stats.cpu_limited_resolution);
3100 EXPECT_FALSE(stats.cpu_limited_framerate);
3101 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3102
3103 // Trigger CPU overuse, should now adapt down.
3104 video_stream_encoder_->TriggerCpuOveruse();
3105 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3106 WaitForEncodedFrame(sequence++);
3107 stats = stats_proxy_->GetStats();
3108 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3109
3110 // Set new degradation preference should clear restrictions since we changed
3111 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003112 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003113 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3114 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3115 WaitForEncodedFrame(sequence++);
3116 stats = stats_proxy_->GetStats();
3117 EXPECT_FALSE(stats.cpu_limited_resolution);
3118 EXPECT_FALSE(stats.cpu_limited_framerate);
3119 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3120
3121 // Force an input frame rate to be available, or the adaptation call won't
3122 // know what framerate to adapt from.
3123 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3124 mock_stats.input_frame_rate = 30;
3125 stats_proxy_->SetMockStats(mock_stats);
3126 video_stream_encoder_->TriggerCpuOveruse();
3127 stats_proxy_->ResetMockStats();
3128 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3129 WaitForEncodedFrame(sequence++);
3130
3131 // We have now adapted once.
3132 stats = stats_proxy_->GetStats();
3133 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3134
3135 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003136 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3137 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003138 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3139 WaitForEncodedFrame(sequence++);
3140 stats = stats_proxy_->GetStats();
3141 EXPECT_FALSE(stats.cpu_limited_resolution);
3142 EXPECT_FALSE(stats.cpu_limited_framerate);
3143 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3144
3145 video_stream_encoder_->Stop();
3146}
3147
3148TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003149 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003151 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003152
asapersson0944a802017-04-07 00:57:58 -07003153 const int kWidth = 1280;
3154 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003155 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003156
asaperssonfab67072017-04-04 05:51:49 -07003157 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003158 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003159 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003160 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003161 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003162 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3163
asapersson02465b82017-04-10 01:12:52 -07003164 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003165 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003166 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003167 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003168 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003169 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003170 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003171 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3172
3173 // Set new source with adaptation still enabled.
3174 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003175 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003176 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003177
3178 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003179 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003180 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003181 stats = stats_proxy_->GetStats();
3182 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003183 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003184 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3185
sprangc5d62e22017-04-02 23:53:04 -07003186 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003188 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003189 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003190 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003191 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003192 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003193 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003194 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003195 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003196 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3197
sprangc5d62e22017-04-02 23:53:04 -07003198 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003199 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003200 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3201 mock_stats.input_frame_rate = 30;
3202 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003203 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003204 stats_proxy_->ResetMockStats();
3205
3206 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003207 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003208 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003209
3210 // Framerate now adapted.
3211 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003212 EXPECT_FALSE(stats.cpu_limited_resolution);
3213 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003214 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3215
3216 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003217 video_stream_encoder_->SetSource(&new_video_source,
3218 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003219 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003220 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003221 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003222
3223 stats = stats_proxy_->GetStats();
3224 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003225 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003226 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3227
3228 // Try to trigger overuse. Should not succeed.
3229 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003230 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003231 stats_proxy_->ResetMockStats();
3232
3233 stats = stats_proxy_->GetStats();
3234 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003235 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003236 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3237
3238 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003239 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003240 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003241 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003242 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003243 stats = stats_proxy_->GetStats();
3244 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003245 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003246 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003247
3248 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003249 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003250 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003251 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003252 stats = stats_proxy_->GetStats();
3253 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003254 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003255 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3256
3257 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003258 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003259 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003260 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003261 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003262 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003263 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003264 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003265 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003266 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003267 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3268
3269 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003270 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003271 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003272 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003273 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003274 stats = stats_proxy_->GetStats();
3275 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003276 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003277 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003278 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003279
mflodmancc3d4422017-08-03 08:27:51 -07003280 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003281}
3282
mflodmancc3d4422017-08-03 08:27:51 -07003283TEST_F(VideoStreamEncoderTest,
3284 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003285 const int kWidth = 1280;
3286 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003288 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003289
asaperssonfab67072017-04-04 05:51:49 -07003290 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003291 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003292
asaperssonfab67072017-04-04 05:51:49 -07003293 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003294 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003295
asaperssonfab67072017-04-04 05:51:49 -07003296 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003297 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003298
asaperssonfab67072017-04-04 05:51:49 -07003299 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003300 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003301
kthelgason876222f2016-11-29 01:44:11 -08003302 // Expect a scale down.
3303 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003304 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003305
asapersson02465b82017-04-10 01:12:52 -07003306 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003307 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003308 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003309 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003310
asaperssonfab67072017-04-04 05:51:49 -07003311 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003312 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003313 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003314 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003315
asaperssonfab67072017-04-04 05:51:49 -07003316 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003317 EXPECT_EQ(std::numeric_limits<int>::max(),
3318 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003319
asaperssonfab67072017-04-04 05:51:49 -07003320 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003321 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003322 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003323 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003324
asapersson02465b82017-04-10 01:12:52 -07003325 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003326 EXPECT_EQ(std::numeric_limits<int>::max(),
3327 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003328
mflodmancc3d4422017-08-03 08:27:51 -07003329 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003330}
3331
mflodmancc3d4422017-08-03 08:27:51 -07003332TEST_F(VideoStreamEncoderTest,
3333 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003334 const int kWidth = 1280;
3335 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003337 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003338
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003339 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003340 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003341 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003342 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003343
3344 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003345 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003346 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003347 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3348 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3349
3350 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003351 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003352 EXPECT_THAT(source.sink_wants(),
3353 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003354 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3355 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3356 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3357
3358 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003359 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003360 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3361 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3362 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3363
mflodmancc3d4422017-08-03 08:27:51 -07003364 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003365}
3366
mflodmancc3d4422017-08-03 08:27:51 -07003367TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003368 const int kWidth = 1280;
3369 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003371 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003372
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003373 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003374 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003375 video_stream_encoder_->SetSource(&source,
3376 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003377 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3378 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003379 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003380
3381 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003383 EXPECT_THAT(source.sink_wants(),
3384 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003385 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3386 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3387 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3388
3389 // Trigger adapt down for same input resolution, expect no change.
3390 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3391 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003393 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3396
3397 // Trigger adapt down for larger input resolution, expect no change.
3398 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3399 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003400 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003401 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3402 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3403 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3404
mflodmancc3d4422017-08-03 08:27:51 -07003405 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003406}
3407
mflodmancc3d4422017-08-03 08:27:51 -07003408TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003409 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3410 const int kWidth = 640;
3411 const int kHeight = 360;
3412 const int64_t kFrameIntervalMs = 150;
3413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003414 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003415
3416 // Enable BALANCED preference, no initial limitation.
3417 AdaptingFrameForwarder source(&time_controller_);
3418 source.set_adaptation_enabled(true);
3419 video_stream_encoder_->SetSource(&source,
3420 webrtc::DegradationPreference::BALANCED);
3421
3422 int64_t timestamp_ms = kFrameIntervalMs;
3423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3424 sink_.WaitForEncodedFrame(kWidth, kHeight);
3425 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3428 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
3430 // Trigger adapt down, expect reduced fps (640x360@15fps).
3431 video_stream_encoder_->TriggerQualityLow();
3432 timestamp_ms += kFrameIntervalMs;
3433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3434 sink_.WaitForEncodedFrame(timestamp_ms);
3435 EXPECT_THAT(source.sink_wants(),
3436 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3439 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3440
3441 // Source requests 270p, expect reduced resolution (480x270@15fps).
3442 source.OnOutputFormatRequest(480, 270);
3443 timestamp_ms += kFrameIntervalMs;
3444 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3445 WaitForEncodedFrame(480, 270);
3446 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3447
3448 // Trigger adapt down, expect reduced fps (480x270@10fps).
3449 video_stream_encoder_->TriggerQualityLow();
3450 timestamp_ms += kFrameIntervalMs;
3451 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3452 sink_.WaitForEncodedFrame(timestamp_ms);
3453 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3454 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3455 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3456 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3457
3458 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3459 source.OnOutputFormatRequest(320, 180);
3460 timestamp_ms += kFrameIntervalMs;
3461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3462 WaitForEncodedFrame(320, 180);
3463 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3464
3465 // Trigger adapt down, expect reduced fps (320x180@7fps).
3466 video_stream_encoder_->TriggerQualityLow();
3467 timestamp_ms += kFrameIntervalMs;
3468 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3469 sink_.WaitForEncodedFrame(timestamp_ms);
3470 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3471 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3473 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3474
3475 // Source requests VGA, expect increased resolution (640x360@7fps).
3476 source.OnOutputFormatRequest(640, 360);
3477 timestamp_ms += kFrameIntervalMs;
3478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3479 WaitForEncodedFrame(timestamp_ms);
3480 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3481
3482 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3483 video_stream_encoder_->TriggerQualityHigh();
3484 timestamp_ms += kFrameIntervalMs;
3485 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3486 WaitForEncodedFrame(timestamp_ms);
3487 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3488 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3490 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3491
3492 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3493 video_stream_encoder_->TriggerQualityHigh();
3494 timestamp_ms += kFrameIntervalMs;
3495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3496 WaitForEncodedFrame(timestamp_ms);
3497 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3499 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3500 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3501
3502 // Trigger adapt up, expect increased fps (640x360@maxfps).
3503 video_stream_encoder_->TriggerQualityHigh();
3504 timestamp_ms += kFrameIntervalMs;
3505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3506 WaitForEncodedFrame(timestamp_ms);
3507 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3508 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3509 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3510 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3511
3512 video_stream_encoder_->Stop();
3513}
3514
3515TEST_F(VideoStreamEncoderTest,
3516 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3517 const int kWidth = 1280;
3518 const int kHeight = 720;
3519 const int64_t kFrameIntervalMs = 150;
3520 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003521 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003522
3523 // Enable BALANCED preference, no initial limitation.
3524 AdaptingFrameForwarder source(&time_controller_);
3525 source.set_adaptation_enabled(true);
3526 video_stream_encoder_->SetSource(&source,
3527 webrtc::DegradationPreference::BALANCED);
3528
3529 int64_t timestamp_ms = kFrameIntervalMs;
3530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3531 sink_.WaitForEncodedFrame(kWidth, kHeight);
3532 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3533 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3535 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3536
3537 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3538 video_stream_encoder_->TriggerQualityLow();
3539 timestamp_ms += kFrameIntervalMs;
3540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3541 sink_.WaitForEncodedFrame(timestamp_ms);
3542 EXPECT_THAT(source.sink_wants(),
3543 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3546 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3547
3548 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3549 video_stream_encoder_->TriggerQualityLow();
3550 timestamp_ms += kFrameIntervalMs;
3551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3552 sink_.WaitForEncodedFrame(timestamp_ms);
3553 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3556 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3557
3558 // Trigger adapt down, expect reduced fps (640x360@15fps).
3559 video_stream_encoder_->TriggerQualityLow();
3560 timestamp_ms += kFrameIntervalMs;
3561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3562 WaitForEncodedFrame(timestamp_ms);
3563 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3566 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3567
3568 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3569 source.OnOutputFormatRequest(320, 180);
3570 timestamp_ms += kFrameIntervalMs;
3571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3572 WaitForEncodedFrame(320, 180);
3573 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3574 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3575
3576 // Trigger adapt down, expect reduced fps (320x180@7fps).
3577 video_stream_encoder_->TriggerCpuOveruse();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3584 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3585 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588
3589 // Source requests HD, expect increased resolution (640x360@7fps).
3590 source.OnOutputFormatRequest(1280, 720);
3591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3593 WaitForEncodedFrame(timestamp_ms);
3594 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3595 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3596
3597 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3598 video_stream_encoder_->TriggerCpuUnderuse();
3599 timestamp_ms += kFrameIntervalMs;
3600 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3601 WaitForEncodedFrame(timestamp_ms);
3602 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3605 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3606 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3607 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3608 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3609
3610 // Trigger adapt up, expect increased fps (640x360@maxfps).
3611 video_stream_encoder_->TriggerQualityHigh();
3612 video_stream_encoder_->TriggerCpuUnderuse();
3613 timestamp_ms += kFrameIntervalMs;
3614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3615 WaitForEncodedFrame(timestamp_ms);
3616 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3618 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3619 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3620 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3621 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3622 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3623
3624 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3625 video_stream_encoder_->TriggerQualityHigh();
3626 video_stream_encoder_->TriggerCpuUnderuse();
3627 timestamp_ms += kFrameIntervalMs;
3628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3629 WaitForEncodedFrame(timestamp_ms);
3630 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3632 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3633 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3634 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3635 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3636 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3637
3638 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3639 video_stream_encoder_->TriggerQualityHigh();
3640 video_stream_encoder_->TriggerCpuUnderuse();
3641 timestamp_ms += kFrameIntervalMs;
3642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3643 WaitForEncodedFrame(timestamp_ms);
3644 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3645 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3646 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3647 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3648 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3649 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3650 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3651
3652 video_stream_encoder_->Stop();
3653}
3654
3655TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003656 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003657 const int kWidth = 1280;
3658 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003660 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003661
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003662 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003663 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003664 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003665 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003666
3667 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003668 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003669 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003670 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3671 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3672
3673 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003674 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003676 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3677 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3678
mflodmancc3d4422017-08-03 08:27:51 -07003679 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003680}
3681
mflodmancc3d4422017-08-03 08:27:51 -07003682TEST_F(VideoStreamEncoderTest,
3683 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003684 const int kWidth = 1280;
3685 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003687 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003688
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003689 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003690 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003691 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003692 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003693
3694 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003695 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003696 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003697 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003698 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3699
3700 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003701 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003702 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003703 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003704 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3705
mflodmancc3d4422017-08-03 08:27:51 -07003706 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003707}
3708
mflodmancc3d4422017-08-03 08:27:51 -07003709TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003710 const int kWidth = 1280;
3711 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003713 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003714
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003715 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003716 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003717 video_stream_encoder_->SetSource(&source,
3718 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003719
3720 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3721 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003722 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3724 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3725 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3726
3727 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003728 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3733
mflodmancc3d4422017-08-03 08:27:51 -07003734 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003735}
3736
mflodmancc3d4422017-08-03 08:27:51 -07003737TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003738 const int kWidth = 1280;
3739 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003741 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003742
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003743 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003744 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003745 video_stream_encoder_->SetSource(&source,
3746 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003747
3748 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3749 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003750 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003751 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3753 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3754
3755 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003756 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003758 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3759 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3760 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003763}
3764
mflodmancc3d4422017-08-03 08:27:51 -07003765TEST_F(VideoStreamEncoderTest,
3766 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003767 const int kWidth = 1280;
3768 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003770 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003771
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003772 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003773 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003774 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003775 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003776 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003777
3778 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003779 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003780 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3782 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3783
3784 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003785 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003786 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003787 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003788 EXPECT_THAT(source.sink_wants(),
3789 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003790 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3791 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3792
3793 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003794 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3797 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3798 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3799
mflodmancc3d4422017-08-03 08:27:51 -07003800 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003801}
3802
mflodmancc3d4422017-08-03 08:27:51 -07003803TEST_F(VideoStreamEncoderTest,
3804 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003805 const int kWidth = 1280;
3806 const int kHeight = 720;
3807 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003809 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003810
3811 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3812 stats.input_frame_rate = kInputFps;
3813 stats_proxy_->SetMockStats(stats);
3814
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003815 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003816 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3817 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003818 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003819
3820 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003821 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003822 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3823 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003824 EXPECT_THAT(video_source_.sink_wants(),
3825 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003826
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003827 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003828 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003829 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003830 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003831 // Give the encoder queue time to process the change in degradation preference
3832 // by waiting for an encoded frame.
3833 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3834 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003835 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003836
3837 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003838 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003839 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3840 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003841 EXPECT_THAT(new_video_source.sink_wants(),
3842 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003843
3844 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003845 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003846 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003847
mflodmancc3d4422017-08-03 08:27:51 -07003848 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003849}
3850
mflodmancc3d4422017-08-03 08:27:51 -07003851TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003852 const int kWidth = 1280;
3853 const int kHeight = 720;
3854 const size_t kNumFrames = 10;
3855
Henrik Boström381d1092020-05-12 18:49:07 +02003856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003857 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003858
asaperssond0de2952017-04-21 01:47:31 -07003859 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003860 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003861 video_source_.set_adaptation_enabled(true);
3862
3863 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3864 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3865
3866 int downscales = 0;
3867 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003868 video_source_.IncomingCapturedFrame(
3869 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3870 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003871
asaperssonfab67072017-04-04 05:51:49 -07003872 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003873 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003874 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003875 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003876
3877 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3878 ++downscales;
3879
3880 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3881 EXPECT_EQ(downscales,
3882 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3883 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003884 }
mflodmancc3d4422017-08-03 08:27:51 -07003885 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003886}
3887
mflodmancc3d4422017-08-03 08:27:51 -07003888TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003889 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3890 const int kWidth = 1280;
3891 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003893 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003894
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003895 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003896 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003897 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003898 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003899 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003900
Åsa Persson8c1bf952018-09-13 10:42:19 +02003901 int64_t timestamp_ms = kFrameIntervalMs;
3902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003903 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003904 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003905 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3906 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3907
3908 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003909 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003910 timestamp_ms += kFrameIntervalMs;
3911 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3912 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003913 EXPECT_THAT(source.sink_wants(),
3914 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003915 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3916 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3917
3918 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003919 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003920 timestamp_ms += kFrameIntervalMs;
3921 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003922 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003923 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003924 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3925 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3926
3927 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003928 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003929 timestamp_ms += kFrameIntervalMs;
3930 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3931 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003932 EXPECT_THAT(source.sink_wants(),
3933 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003934 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3935 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3936
3937 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003938 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003939 timestamp_ms += kFrameIntervalMs;
3940 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003941 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003942 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003943 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3944 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3945
mflodmancc3d4422017-08-03 08:27:51 -07003946 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003947}
3948
mflodmancc3d4422017-08-03 08:27:51 -07003949TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003950 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3951 const int kWidth = 1280;
3952 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003953 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003954 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003955
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003956 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003957 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003958 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003959 video_stream_encoder_->SetSource(&source,
3960 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003961
Åsa Persson8c1bf952018-09-13 10:42:19 +02003962 int64_t timestamp_ms = kFrameIntervalMs;
3963 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003964 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003965 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003966 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3967 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3968
3969 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003970 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003971 timestamp_ms += kFrameIntervalMs;
3972 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3973 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003974 EXPECT_THAT(source.sink_wants(),
3975 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003976 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3977 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3978
3979 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003980 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003981 timestamp_ms += kFrameIntervalMs;
3982 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003983 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003984 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003985 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3986 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3987
3988 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003989 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003990 timestamp_ms += kFrameIntervalMs;
3991 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3992 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003993 EXPECT_THAT(source.sink_wants(),
3994 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003995 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3996 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3997
3998 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003999 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004000 timestamp_ms += kFrameIntervalMs;
4001 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004002 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004003 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4005 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4006
mflodmancc3d4422017-08-03 08:27:51 -07004007 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004008}
4009
Sergey Silkin41c650b2019-10-14 13:12:19 +02004010TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4011 fake_encoder_.SetResolutionBitrateLimits(
4012 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4013
Henrik Boström381d1092020-05-12 18:49:07 +02004014 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004015 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4016 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4017 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4018 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004019
4020 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004021 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004022 source.set_adaptation_enabled(true);
4023 video_stream_encoder_->SetSource(
4024 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4025
4026 // Insert 720p frame.
4027 int64_t timestamp_ms = kFrameIntervalMs;
4028 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4029 WaitForEncodedFrame(1280, 720);
4030
4031 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004033 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4034 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4035 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4036 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004037 video_stream_encoder_->TriggerQualityLow();
4038
4039 // Insert 720p frame. It should be downscaled and encoded.
4040 timestamp_ms += kFrameIntervalMs;
4041 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4042 WaitForEncodedFrame(960, 540);
4043
4044 // Trigger adapt up. Higher resolution should not be requested duo to lack
4045 // of bitrate.
4046 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004047 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004048
4049 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004051 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4052 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4053 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4054 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004055
4056 // Trigger adapt up. Higher resolution should be requested.
4057 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004058 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004059
4060 video_stream_encoder_->Stop();
4061}
4062
4063TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4064 fake_encoder_.SetResolutionBitrateLimits(
4065 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4066
4067 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004069 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4070 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4071 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4072 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004073
4074 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004075 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004076 source.set_adaptation_enabled(true);
4077 video_stream_encoder_->SetSource(
4078 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4079
4080 // Insert 720p frame. It should be dropped and lower resolution should be
4081 // requested.
4082 int64_t timestamp_ms = kFrameIntervalMs;
4083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4084 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004085 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004086
4087 // Insert 720p frame. It should be downscaled and encoded.
4088 timestamp_ms += kFrameIntervalMs;
4089 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4090 WaitForEncodedFrame(960, 540);
4091
4092 video_stream_encoder_->Stop();
4093}
4094
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004095class BalancedDegradationTest : public VideoStreamEncoderTest {
4096 protected:
4097 void SetupTest() {
4098 // Reset encoder for field trials to take effect.
4099 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004100 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004101
4102 // Enable BALANCED preference.
4103 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004104 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4105 }
4106
Asa Persson606d3cb2021-10-04 10:07:11 +02004107 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004109 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004110 }
4111
Åsa Persson45b176f2019-09-30 11:19:05 +02004112 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004113 timestamp_ms_ += kFrameIntervalMs;
4114 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004115 }
4116
4117 void InsertFrameAndWaitForEncoded() {
4118 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004119 sink_.WaitForEncodedFrame(timestamp_ms_);
4120 }
4121
4122 const int kWidth = 640; // pixels:640x360=230400
4123 const int kHeight = 360;
4124 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4125 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004126 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004127};
4128
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004129TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004130 test::ScopedKeyValueConfig field_trials(
4131 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004132 "WebRTC-Video-BalancedDegradationSettings/"
4133 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4134 SetupTest();
4135
4136 // Force input frame rate.
4137 const int kInputFps = 24;
4138 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4139 stats.input_frame_rate = kInputFps;
4140 stats_proxy_->SetMockStats(stats);
4141
Åsa Persson45b176f2019-09-30 11:19:05 +02004142 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004143 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004144
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004145 // Trigger adapt down, expect scaled down framerate and resolution,
4146 // since Fps diff (input-requested:0) < threshold.
4147 video_stream_encoder_->TriggerQualityLow();
4148 EXPECT_THAT(source_.sink_wants(),
4149 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004150
4151 video_stream_encoder_->Stop();
4152}
4153
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004154TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004155 test::ScopedKeyValueConfig field_trials(
4156 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004157 "WebRTC-Video-BalancedDegradationSettings/"
4158 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4159 SetupTest();
4160
4161 // Force input frame rate.
4162 const int kInputFps = 25;
4163 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4164 stats.input_frame_rate = kInputFps;
4165 stats_proxy_->SetMockStats(stats);
4166
Åsa Persson45b176f2019-09-30 11:19:05 +02004167 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004168 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004169
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004170 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4171 // Fps diff (input-requested:1) == threshold.
4172 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004173 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004174
4175 video_stream_encoder_->Stop();
4176}
4177
4178TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004179 test::ScopedKeyValueConfig field_trials(
4180 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004181 "WebRTC-Video-BalancedDegradationSettings/"
4182 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4183 SetupTest();
4184
4185 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4186
Åsa Persson45b176f2019-09-30 11:19:05 +02004187 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004188 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004189
4190 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4191 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004192 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004193
4194 video_stream_encoder_->Stop();
4195}
4196
Åsa Perssonccfb3402019-09-25 15:13:04 +02004197TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004198 test::ScopedKeyValueConfig field_trials(
4199 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004200 "WebRTC-Video-BalancedDegradationSettings/"
4201 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004202 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004203
Asa Persson606d3cb2021-10-04 10:07:11 +02004204 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4205 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4206 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004207
Åsa Persson45b176f2019-09-30 11:19:05 +02004208 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004209 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004210 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4211
4212 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4213 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004214 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004215 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004216 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4217
4218 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4219 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004220 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004221 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004222 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4223
Åsa Persson30ab0152019-08-27 12:22:33 +02004224 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4225 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004226 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004227 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004228 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004229 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4230
4231 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004232 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004233 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004234 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004235
Åsa Persson30ab0152019-08-27 12:22:33 +02004236 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004237 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004238 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004239 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004240 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004241 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4242
4243 video_stream_encoder_->Stop();
4244}
4245
Åsa Perssonccfb3402019-09-25 15:13:04 +02004246TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004247 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004248 test::ScopedKeyValueConfig field_trials(
4249 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004250 "WebRTC-Video-BalancedDegradationSettings/"
4251 "pixels:57600|129600|230400,fps:7|24|24/");
4252 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004253 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004254
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004255 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004256
4257 // Insert frame, expect scaled down:
4258 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4259 InsertFrame();
4260 EXPECT_FALSE(WaitForFrame(1000));
4261 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4262 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4263
4264 // Insert frame, expect scaled down:
4265 // resolution (320x180@24fps).
4266 InsertFrame();
4267 EXPECT_FALSE(WaitForFrame(1000));
4268 EXPECT_LT(source_.sink_wants().max_pixel_count,
4269 source_.last_wants().max_pixel_count);
4270 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4271
4272 // Frame should not be dropped (min pixels per frame reached).
4273 InsertFrameAndWaitForEncoded();
4274
4275 video_stream_encoder_->Stop();
4276}
4277
4278TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004279 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004280 test::ScopedKeyValueConfig field_trials(
4281 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004282 "WebRTC-Video-BalancedDegradationSettings/"
4283 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004284 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004285
Asa Persson606d3cb2021-10-04 10:07:11 +02004286 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4287 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4288 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004289
Åsa Persson45b176f2019-09-30 11:19:05 +02004290 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004291 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004292 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4293
4294 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4295 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004296 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004297 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004298 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4299
4300 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4301 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004302 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004303 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004304 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4305
4306 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4307 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004308 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004309 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004310 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4311
Åsa Persson30ab0152019-08-27 12:22:33 +02004312 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4313 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004314 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004315 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004316 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4317
4318 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4319 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004320 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004321 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4322
4323 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004324 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004325 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004326 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004327 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004328 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4329
4330 video_stream_encoder_->Stop();
4331}
4332
Åsa Perssonccfb3402019-09-25 15:13:04 +02004333TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004334 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004335 test::ScopedKeyValueConfig field_trials(
4336 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004337 "WebRTC-Video-BalancedDegradationSettings/"
4338 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004339 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004340
Asa Persson606d3cb2021-10-04 10:07:11 +02004341 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4342 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4343 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4344 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4345 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004346
Åsa Persson45b176f2019-09-30 11:19:05 +02004347 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004348 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004349 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
4351 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4352 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004353 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004354 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
4357 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4358 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004359 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004360 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004361 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4362
4363 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4364 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004365 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004366 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004367 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4368
4369 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4370 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004371 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004372 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4373
4374 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004375 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004376 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004377 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004378 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004379 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4380
4381 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004382 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004383 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004384 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004385 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4386
4387 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004388 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004389 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004390 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004391 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004392 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4393
Åsa Persson1b247f12019-08-14 17:26:39 +02004394 video_stream_encoder_->Stop();
4395}
4396
mflodmancc3d4422017-08-03 08:27:51 -07004397TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004398 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4399 const int kWidth = 1280;
4400 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004401 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004402 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004403
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004404 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004405 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004406 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004407 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004408 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004409
Åsa Persson8c1bf952018-09-13 10:42:19 +02004410 int64_t timestamp_ms = kFrameIntervalMs;
4411 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004412 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004413 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004414 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4415 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4416 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4417 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4418
4419 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004420 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004421 timestamp_ms += kFrameIntervalMs;
4422 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4423 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004424 EXPECT_THAT(source.sink_wants(),
4425 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004426 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4428 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4429 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4430
4431 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004432 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004433 timestamp_ms += kFrameIntervalMs;
4434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4435 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004436 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004437 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4438 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4439 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4440 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4441
Jonathan Yubc771b72017-12-08 17:04:29 -08004442 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004443 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004444 timestamp_ms += kFrameIntervalMs;
4445 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4446 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004447 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004448 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4449 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004450 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004451 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4452
Jonathan Yubc771b72017-12-08 17:04:29 -08004453 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004454 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004455 timestamp_ms += kFrameIntervalMs;
4456 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4457 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004458 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004459 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004460 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4461 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4462 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4463 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4464
Jonathan Yubc771b72017-12-08 17:04:29 -08004465 // Trigger quality adapt down, expect no change (min resolution reached).
4466 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004467 timestamp_ms += kFrameIntervalMs;
4468 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4469 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004470 EXPECT_THAT(source.sink_wants(), FpsMax());
4471 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004472 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4473 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4474 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4475 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4476
Evan Shrubsole64469032020-06-11 10:45:29 +02004477 // Trigger quality adapt up, expect upscaled resolution (480x270).
4478 video_stream_encoder_->TriggerQualityHigh();
4479 timestamp_ms += kFrameIntervalMs;
4480 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4481 WaitForEncodedFrame(timestamp_ms);
4482 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4483 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4484 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4485 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4486 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4487
4488 // Trigger quality and cpu adapt up since both are most limited, expect
4489 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004490 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004491 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004492 timestamp_ms += kFrameIntervalMs;
4493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4494 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004495 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004496 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4497 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4498 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004499 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004500
Evan Shrubsole64469032020-06-11 10:45:29 +02004501 // Trigger quality and cpu adapt up since both are most limited, expect
4502 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004503 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004504 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004505 timestamp_ms += kFrameIntervalMs;
4506 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4507 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004508 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004509 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004510 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004512 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4513 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004514
Evan Shrubsole64469032020-06-11 10:45:29 +02004515 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4516 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004517 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004518 timestamp_ms += kFrameIntervalMs;
4519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4520 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004521 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004522 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004524 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004525 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004526
4527 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004528 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004529 timestamp_ms += kFrameIntervalMs;
4530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004531 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004532 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004533 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004534 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004536 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004537 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004538
mflodmancc3d4422017-08-03 08:27:51 -07004539 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004540}
4541
mflodmancc3d4422017-08-03 08:27:51 -07004542TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004543 const int kWidth = 640;
4544 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004545
Henrik Boström381d1092020-05-12 18:49:07 +02004546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004547 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004548
perkj803d97f2016-11-01 11:45:46 -07004549 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004550 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004551 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004552 }
4553
mflodmancc3d4422017-08-03 08:27:51 -07004554 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004555 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004556 video_source_.IncomingCapturedFrame(CreateFrame(
4557 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004558 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004559 }
4560
mflodmancc3d4422017-08-03 08:27:51 -07004561 video_stream_encoder_->Stop();
4562 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004563 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004564
Ying Wangef3998f2019-12-09 13:06:53 +01004565 EXPECT_METRIC_EQ(
4566 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4567 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004568 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4569}
4570
mflodmancc3d4422017-08-03 08:27:51 -07004571TEST_F(VideoStreamEncoderTest,
4572 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004573 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004574 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004575 const int kWidth = 640;
4576 const int kHeight = 360;
4577
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004578 video_stream_encoder_->SetSource(&video_source_,
4579 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004580
4581 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4582 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004583 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004584 }
4585
mflodmancc3d4422017-08-03 08:27:51 -07004586 video_stream_encoder_->Stop();
4587 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004588 stats_proxy_.reset();
4589
4590 EXPECT_EQ(0,
4591 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4592}
4593
Per Kjellanderdcef6412020-10-07 15:09:05 +02004594TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4595 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004596 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004597 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004598
4599 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004600 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004601 SimulcastRateAllocator(fake_encoder_.config())
4602 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004603 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004604
Henrik Boström381d1092020-05-12 18:49:07 +02004605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004606 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004607
sprang57c2fff2017-01-16 06:24:02 -08004608 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004609 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4610 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004611 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4612 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4613
Erik Språngd7329ca2019-02-21 21:19:53 +01004614 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004615 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004616 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004617
Per Kjellanderdcef6412020-10-07 15:09:05 +02004618 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004619 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004620 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4621 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004622 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004623 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004624
Per Kjellanderdcef6412020-10-07 15:09:05 +02004625 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004626 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004627 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004628 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004629 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4630 WaitForEncodedFrame(CurrentTimeMs());
4631 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004632 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004633 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004634
mflodmancc3d4422017-08-03 08:27:51 -07004635 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004636}
4637
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004638TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004639 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004640 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004641 kVideoLayersAllocation);
4642
4643 const int kDefaultFps = 30;
4644
4645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004646 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004647
4648 video_source_.IncomingCapturedFrame(
4649 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4650 WaitForEncodedFrame(CurrentTimeMs());
4651 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4652 VideoLayersAllocation last_layer_allocation =
4653 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004654 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004655 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4656
4657 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004658 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004659 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004660 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004661 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4662
Erik Språng9d69cbe2020-10-22 17:44:42 +02004663 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004664 int number_of_layers_allocation = 1;
4665 const int64_t start_time_ms = CurrentTimeMs();
4666 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4667 video_source_.IncomingCapturedFrame(
4668 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4669 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004670 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4671 number_of_layers_allocation = sink_.number_of_layers_allocations();
4672 VideoLayersAllocation new_allocation =
4673 sink_.GetLastVideoLayersAllocation();
4674 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4675 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4676 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4677 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4678 .target_bitrate_per_temporal_layer,
4679 last_layer_allocation.active_spatial_layers[0]
4680 .target_bitrate_per_temporal_layer);
4681 last_layer_allocation = new_allocation;
4682 }
4683 }
4684 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4685 video_stream_encoder_->Stop();
4686}
4687
4688TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004689 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004690 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4691 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4692 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004693 VideoEncoderConfig video_encoder_config;
4694 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4695 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004696 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004697 video_encoder_config.content_type =
4698 VideoEncoderConfig::ContentType::kRealtimeVideo;
4699 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004700 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004701 VideoEncoder::GetDefaultVp8Settings());
4702 for (auto& layer : video_encoder_config.simulcast_layers) {
4703 layer.num_temporal_layers = 2;
4704 }
4705 // Simulcast layers are used for enabling/disabling streams.
4706 video_encoder_config.simulcast_layers[0].active = true;
4707 video_encoder_config.simulcast_layers[1].active = false;
4708 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004709 ConfigureEncoder(std::move(video_encoder_config),
4710 VideoStreamEncoder::BitrateAllocationCallbackType::
4711 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004712
4713 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004714 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004715
4716 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4717 WaitForEncodedFrame(CurrentTimeMs());
4718 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4719 VideoLayersAllocation last_layer_allocation =
4720 sink_.GetLastVideoLayersAllocation();
4721
4722 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4723 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4724 .target_bitrate_per_temporal_layer,
4725 SizeIs(2));
4726 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4727 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4728 video_stream_encoder_->Stop();
4729}
4730
4731TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004732 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004733 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4734 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4735 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004736 VideoEncoderConfig video_encoder_config;
4737 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4738 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004739 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004740 video_encoder_config.content_type =
4741 VideoEncoderConfig::ContentType::kRealtimeVideo;
4742 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004743 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004744 VideoEncoder::GetDefaultVp8Settings());
4745 for (auto& layer : video_encoder_config.simulcast_layers) {
4746 layer.num_temporal_layers = 2;
4747 }
4748 // Simulcast layers are used for enabling/disabling streams.
4749 video_encoder_config.simulcast_layers[0].active = true;
4750 video_encoder_config.simulcast_layers[1].active = false;
4751 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004752 ConfigureEncoder(std::move(video_encoder_config),
4753 VideoStreamEncoder::BitrateAllocationCallbackType::
4754 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755
4756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004757 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004758
4759 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4760 WaitForEncodedFrame(CurrentTimeMs());
4761 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4762 VideoLayersAllocation last_layer_allocation =
4763 sink_.GetLastVideoLayersAllocation();
4764
4765 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4766 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4767 .target_bitrate_per_temporal_layer,
4768 SizeIs(2));
4769 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4770
4771 video_stream_encoder_->Stop();
4772}
4773
4774TEST_F(VideoStreamEncoderTest,
4775 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4776 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4777 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004778 VideoEncoderConfig video_encoder_config;
4779 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4780 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004781 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004782 video_encoder_config.content_type =
4783 VideoEncoderConfig::ContentType::kRealtimeVideo;
4784 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4785 vp9_settings.numberOfSpatialLayers = 2;
4786 vp9_settings.numberOfTemporalLayers = 2;
4787 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4788 vp9_settings.automaticResizeOn = false;
4789 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004790 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004791 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004792 ConfigureEncoder(std::move(video_encoder_config),
4793 VideoStreamEncoder::BitrateAllocationCallbackType::
4794 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004795
4796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004797 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004798
4799 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4800 WaitForEncodedFrame(CurrentTimeMs());
4801 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4802 VideoLayersAllocation last_layer_allocation =
4803 sink_.GetLastVideoLayersAllocation();
4804
4805 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4806 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4807 .target_bitrate_per_temporal_layer,
4808 SizeIs(2));
4809 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4810 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4811 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4812 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4813 .target_bitrate_per_temporal_layer,
4814 SizeIs(2));
4815 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4816 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4817 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4818
4819 // Since full SVC is used, expect the top layer to utilize the full target
4820 // rate.
4821 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4822 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004823 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004824 video_stream_encoder_->Stop();
4825}
4826
4827TEST_F(VideoStreamEncoderTest,
4828 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4829 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4830 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004831 VideoEncoderConfig video_encoder_config;
4832 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4833 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004834 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004835 video_encoder_config.content_type =
4836 VideoEncoderConfig::ContentType::kRealtimeVideo;
4837 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4838 vp9_settings.numberOfSpatialLayers = 2;
4839 vp9_settings.numberOfTemporalLayers = 2;
4840 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4841 vp9_settings.automaticResizeOn = false;
4842 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004843 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004844 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004845 ConfigureEncoder(std::move(video_encoder_config),
4846 VideoStreamEncoder::BitrateAllocationCallbackType::
4847 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004848
4849 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004850 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851
4852 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4853 WaitForEncodedFrame(CurrentTimeMs());
4854 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4855 VideoLayersAllocation last_layer_allocation =
4856 sink_.GetLastVideoLayersAllocation();
4857
4858 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4859 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4860 .target_bitrate_per_temporal_layer,
4861 SizeIs(1));
4862 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4863 .target_bitrate_per_temporal_layer,
4864 SizeIs(1));
4865 // Since full SVC is used, expect the top layer to utilize the full target
4866 // rate.
4867 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4868 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004869 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004870 video_stream_encoder_->Stop();
4871}
4872
4873TEST_F(VideoStreamEncoderTest,
4874 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4875 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4876 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004877 VideoEncoderConfig video_encoder_config;
4878 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4879 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004880 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004881 video_encoder_config.content_type =
4882 VideoEncoderConfig::ContentType::kRealtimeVideo;
4883 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4884 vp9_settings.numberOfSpatialLayers = 2;
4885 vp9_settings.numberOfTemporalLayers = 2;
4886 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4887 vp9_settings.automaticResizeOn = false;
4888 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004889 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004890 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004891 ConfigureEncoder(std::move(video_encoder_config),
4892 VideoStreamEncoder::BitrateAllocationCallbackType::
4893 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004894
4895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004897
4898 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4899 WaitForEncodedFrame(CurrentTimeMs());
4900 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4901 VideoLayersAllocation last_layer_allocation =
4902 sink_.GetLastVideoLayersAllocation();
4903
4904 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4905 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4906 .target_bitrate_per_temporal_layer,
4907 SizeIs(2));
4908 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4909 .target_bitrate_per_temporal_layer,
4910 SizeIs(2));
4911 // Since KSVC is, spatial layers are independend except on key frames.
4912 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4913 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004914 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004915 video_stream_encoder_->Stop();
4916}
4917
4918TEST_F(VideoStreamEncoderTest,
4919 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4920 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4921 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4922 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004923 VideoEncoderConfig video_encoder_config;
4924 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4925 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004926 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004927 video_encoder_config.content_type =
4928 VideoEncoderConfig::ContentType::kRealtimeVideo;
4929 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4930 vp9_settings.numberOfSpatialLayers = 3;
4931 vp9_settings.numberOfTemporalLayers = 2;
4932 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4933 vp9_settings.automaticResizeOn = false;
4934 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004935 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004936 vp9_settings);
4937 // Simulcast layers are used for enabling/disabling streams.
4938 video_encoder_config.simulcast_layers.resize(3);
4939 video_encoder_config.simulcast_layers[0].active = false;
4940 video_encoder_config.simulcast_layers[1].active = true;
4941 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004942 ConfigureEncoder(std::move(video_encoder_config),
4943 VideoStreamEncoder::BitrateAllocationCallbackType::
4944 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004945
4946 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004947 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004948
4949 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4950 WaitForEncodedFrame(CurrentTimeMs());
4951 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4952 VideoLayersAllocation last_layer_allocation =
4953 sink_.GetLastVideoLayersAllocation();
4954
4955 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4956 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4957 .target_bitrate_per_temporal_layer,
4958 SizeIs(2));
4959 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4960 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4961
4962 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4963 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4964 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4965 .target_bitrate_per_temporal_layer,
4966 SizeIs(2));
4967 // Since full SVC is used, expect the top layer to utilize the full target
4968 // rate.
4969 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4970 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004971 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004972 video_stream_encoder_->Stop();
4973}
4974
4975TEST_F(VideoStreamEncoderTest,
4976 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4977 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4978 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4979 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004980 VideoEncoderConfig video_encoder_config;
4981 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4982 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004983 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004984 video_encoder_config.content_type =
4985 VideoEncoderConfig::ContentType::kRealtimeVideo;
4986 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4987 vp9_settings.numberOfSpatialLayers = 3;
4988 vp9_settings.numberOfTemporalLayers = 2;
4989 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4990 vp9_settings.automaticResizeOn = false;
4991 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004992 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004993 vp9_settings);
4994 // Simulcast layers are used for enabling/disabling streams.
4995 video_encoder_config.simulcast_layers.resize(3);
4996 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004997 ConfigureEncoder(std::move(video_encoder_config),
4998 VideoStreamEncoder::BitrateAllocationCallbackType::
4999 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000
5001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005002 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005003
5004 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5005 WaitForEncodedFrame(CurrentTimeMs());
5006 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5007 VideoLayersAllocation last_layer_allocation =
5008 sink_.GetLastVideoLayersAllocation();
5009
5010 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5011 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5012 .target_bitrate_per_temporal_layer,
5013 SizeIs(2));
5014 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5015 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5016
5017 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5018 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5019 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5020 .target_bitrate_per_temporal_layer,
5021 SizeIs(2));
5022 video_stream_encoder_->Stop();
5023}
5024
5025TEST_F(VideoStreamEncoderTest,
5026 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5027 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5028 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5029 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005030 VideoEncoderConfig video_encoder_config;
5031 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5032 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005033 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005034 video_encoder_config.content_type =
5035 VideoEncoderConfig::ContentType::kRealtimeVideo;
5036 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5037 vp9_settings.numberOfSpatialLayers = 3;
5038 vp9_settings.numberOfTemporalLayers = 2;
5039 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5040 vp9_settings.automaticResizeOn = false;
5041 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005042 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005043 vp9_settings);
5044 // Simulcast layers are used for enabling/disabling streams.
5045 video_encoder_config.simulcast_layers.resize(3);
5046 video_encoder_config.simulcast_layers[0].active = false;
5047 video_encoder_config.simulcast_layers[1].active = false;
5048 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005049 ConfigureEncoder(std::move(video_encoder_config),
5050 VideoStreamEncoder::BitrateAllocationCallbackType::
5051 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005052
5053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005054 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005055
5056 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5057 WaitForEncodedFrame(CurrentTimeMs());
5058 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5059 VideoLayersAllocation last_layer_allocation =
5060 sink_.GetLastVideoLayersAllocation();
5061
5062 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5063 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5064 .target_bitrate_per_temporal_layer,
5065 SizeIs(2));
5066 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5067 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5068 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5069 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005070 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005071 video_stream_encoder_->Stop();
5072}
5073
5074TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5075 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005076 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005077 kVideoLayersAllocation);
5078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005079 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005080
5081 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5082 WaitForEncodedFrame(CurrentTimeMs());
5083 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5084 VideoLayersAllocation last_layer_allocation =
5085 sink_.GetLastVideoLayersAllocation();
5086
5087 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5088 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5089 .target_bitrate_per_temporal_layer,
5090 SizeIs(1));
5091 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5092 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005093 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005094 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5095 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5096 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5097 video_stream_encoder_->Stop();
5098}
5099
5100TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005101 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5102 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005103 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005104 kVideoLayersAllocation);
5105
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005107 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005108
5109 video_source_.IncomingCapturedFrame(
5110 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5111 WaitForEncodedFrame(CurrentTimeMs());
5112 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5113 VideoLayersAllocation last_layer_allocation =
5114 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005115 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005116 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5117 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5118 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005119 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005120
5121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005122 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5123 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005124 video_source_.IncomingCapturedFrame(
5125 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5126 WaitForEncodedFrame(CurrentTimeMs());
5127
5128 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5129 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5130 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5131 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5132 .target_bitrate_per_temporal_layer[0],
5133 DataRate::Zero());
5134
5135 video_stream_encoder_->Stop();
5136}
5137
Per Kjellander4190ce92020-12-15 17:24:55 +01005138TEST_F(VideoStreamEncoderTest,
5139 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5140 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005141 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005142 kVideoLayersAllocation);
5143
5144 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005145 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5146 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005147
5148 video_source_.IncomingCapturedFrame(
5149 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5150 WaitForEncodedFrame(CurrentTimeMs());
5151 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5152 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5153 SizeIs(2));
5154 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5155 codec_width_);
5156 EXPECT_EQ(
5157 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5158 codec_height_);
5159
5160 video_source_.IncomingCapturedFrame(
5161 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5162 WaitForEncodedFrame(CurrentTimeMs());
5163 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5164 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5165 SizeIs(2));
5166 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5167 codec_width_ / 2);
5168 EXPECT_EQ(
5169 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5170 codec_height_ / 2);
5171
5172 video_stream_encoder_->Stop();
5173}
5174
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005175TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5176 // 2 TLs configured, temporal layers supported by encoder.
5177 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005178 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005179 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005180 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005181 fake_encoder_.SetTemporalLayersSupported(0, true);
5182
5183 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005184 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005185 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005186 kNumTemporalLayers, /*temporal_id*/ 0,
5187 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005188 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005189 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005190 kNumTemporalLayers, /*temporal_id*/ 1,
5191 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005192 VideoBitrateAllocation expected_bitrate;
5193 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5194 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5195
5196 VerifyAllocatedBitrate(expected_bitrate);
5197 video_stream_encoder_->Stop();
5198}
5199
5200TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5201 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005202 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005203 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005204 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005205 fake_encoder_.SetTemporalLayersSupported(0, false);
5206
5207 // Temporal layers not supported by the encoder.
5208 // Total bitrate should be at ti:0.
5209 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005210 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005211
5212 VerifyAllocatedBitrate(expected_bitrate);
5213 video_stream_encoder_->Stop();
5214}
5215
5216TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005217 webrtc::test::ScopedKeyValueConfig field_trials(
5218 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005219 "WebRTC-Video-QualityScalerSettings/"
5220 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5221 // Reset encoder for field trials to take effect.
5222 ConfigureEncoder(video_encoder_config_.Copy());
5223
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005224 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005225 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005226 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005227 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005228 fake_encoder_.SetTemporalLayersSupported(0, true);
5229 fake_encoder_.SetTemporalLayersSupported(1, false);
5230
5231 const int kS0Bps = 150000;
5232 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005233 kS0Bps *
5234 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5235 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005236 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005237 kS0Bps *
5238 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5239 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005240 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005241 // Temporal layers not supported by si:1.
5242 VideoBitrateAllocation expected_bitrate;
5243 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5244 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5245 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5246
5247 VerifyAllocatedBitrate(expected_bitrate);
5248 video_stream_encoder_->Stop();
5249}
5250
Niels Möller7dc26b72017-12-06 10:27:48 +01005251TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5252 const int kFrameWidth = 1280;
5253 const int kFrameHeight = 720;
5254 const int kFramerate = 24;
5255
Henrik Boström381d1092020-05-12 18:49:07 +02005256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005257 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005258 test::FrameForwarder source;
5259 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005260 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005261
5262 // Insert a single frame, triggering initial configuration.
5263 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5264 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5265
5266 EXPECT_EQ(
5267 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5268 kDefaultFramerate);
5269
5270 // Trigger reconfigure encoder (without resetting the entire instance).
5271 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005272 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5273 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005274 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005275 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005276 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005277 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5278
5279 // Detector should be updated with fps limit from codec config.
5280 EXPECT_EQ(
5281 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5282 kFramerate);
5283
5284 // Trigger overuse, max framerate should be reduced.
5285 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5286 stats.input_frame_rate = kFramerate;
5287 stats_proxy_->SetMockStats(stats);
5288 video_stream_encoder_->TriggerCpuOveruse();
5289 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5290 int adapted_framerate =
5291 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5292 EXPECT_LT(adapted_framerate, kFramerate);
5293
5294 // Trigger underuse, max framerate should go back to codec configured fps.
5295 // Set extra low fps, to make sure it's actually reset, not just incremented.
5296 stats = stats_proxy_->GetStats();
5297 stats.input_frame_rate = adapted_framerate / 2;
5298 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005299 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005300 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5301 EXPECT_EQ(
5302 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5303 kFramerate);
5304
5305 video_stream_encoder_->Stop();
5306}
5307
5308TEST_F(VideoStreamEncoderTest,
5309 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5310 const int kFrameWidth = 1280;
5311 const int kFrameHeight = 720;
5312 const int kLowFramerate = 15;
5313 const int kHighFramerate = 25;
5314
Henrik Boström381d1092020-05-12 18:49:07 +02005315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005316 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005317 test::FrameForwarder source;
5318 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005319 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005320
5321 // Trigger initial configuration.
5322 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005323 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5324 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005325 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005326 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005327 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005328 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005329 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5330
5331 EXPECT_EQ(
5332 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5333 kLowFramerate);
5334
5335 // Trigger overuse, max framerate should be reduced.
5336 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5337 stats.input_frame_rate = kLowFramerate;
5338 stats_proxy_->SetMockStats(stats);
5339 video_stream_encoder_->TriggerCpuOveruse();
5340 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5341 int adapted_framerate =
5342 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5343 EXPECT_LT(adapted_framerate, kLowFramerate);
5344
5345 // Reconfigure the encoder with a new (higher max framerate), max fps should
5346 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005347 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005348 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5349 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005350 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005351 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5352
5353 EXPECT_EQ(
5354 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5355 adapted_framerate);
5356
5357 // Trigger underuse, max framerate should go back to codec configured fps.
5358 stats = stats_proxy_->GetStats();
5359 stats.input_frame_rate = adapted_framerate;
5360 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005361 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005362 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5363 EXPECT_EQ(
5364 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5365 kHighFramerate);
5366
5367 video_stream_encoder_->Stop();
5368}
5369
mflodmancc3d4422017-08-03 08:27:51 -07005370TEST_F(VideoStreamEncoderTest,
5371 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005372 const int kFrameWidth = 1280;
5373 const int kFrameHeight = 720;
5374 const int kFramerate = 24;
5375
Henrik Boström381d1092020-05-12 18:49:07 +02005376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005377 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005378 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005379 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005380 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005381
5382 // Trigger initial configuration.
5383 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005384 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5385 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005386 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005387 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005388 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005389 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005390 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005391
Niels Möller7dc26b72017-12-06 10:27:48 +01005392 EXPECT_EQ(
5393 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5394 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005395
5396 // Trigger overuse, max framerate should be reduced.
5397 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5398 stats.input_frame_rate = kFramerate;
5399 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005400 video_stream_encoder_->TriggerCpuOveruse();
5401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005402 int adapted_framerate =
5403 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005404 EXPECT_LT(adapted_framerate, kFramerate);
5405
5406 // Change degradation preference to not enable framerate scaling. Target
5407 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005408 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005409 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005410 EXPECT_EQ(
5411 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5412 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005413
mflodmancc3d4422017-08-03 08:27:51 -07005414 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005415}
5416
mflodmancc3d4422017-08-03 08:27:51 -07005417TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005418 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005419 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005420 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5421 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5422 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005423 const int kWidth = 640;
5424 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005425
asaperssonfab67072017-04-04 05:51:49 -07005426 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005427
5428 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005429 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005430
5431 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005432 EXPECT_TRUE_WAIT(
5433 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005434
sprangc5d62e22017-04-02 23:53:04 -07005435 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005436
asaperssonfab67072017-04-04 05:51:49 -07005437 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005438 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005439 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005440
5441 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005442 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005443
Henrik Boström2671dac2020-05-19 16:29:09 +02005444 EXPECT_TRUE_WAIT(
5445 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005446
mflodmancc3d4422017-08-03 08:27:51 -07005447 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005448}
5449
mflodmancc3d4422017-08-03 08:27:51 -07005450TEST_F(VideoStreamEncoderTest,
5451 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005452 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005454 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5455 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5456 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005457 const int kWidth = 640;
5458 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005459
5460 // We expect the n initial frames to get dropped.
5461 int i;
5462 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005463 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005464 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005465 }
5466 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005467 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005468 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005469
5470 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005471 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005472
mflodmancc3d4422017-08-03 08:27:51 -07005473 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005474}
5475
mflodmancc3d4422017-08-03 08:27:51 -07005476TEST_F(VideoStreamEncoderTest,
5477 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005478 const int kWidth = 640;
5479 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005481 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005482
5483 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005484 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005485 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005486
asaperssonfab67072017-04-04 05:51:49 -07005487 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005488 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005489 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005490
mflodmancc3d4422017-08-03 08:27:51 -07005491 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005492}
5493
mflodmancc3d4422017-08-03 08:27:51 -07005494TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005495 const int kWidth = 640;
5496 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005497 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005498
5499 VideoEncoderConfig video_encoder_config;
5500 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5501 // Make format different, to force recreation of encoder.
5502 video_encoder_config.video_format.parameters["foo"] = "foo";
5503 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005504 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005506 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005507
kthelgasonb83797b2017-02-14 11:57:25 -08005508 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005509 video_stream_encoder_->SetSource(&video_source_,
5510 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005511
asaperssonfab67072017-04-04 05:51:49 -07005512 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005513 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005514 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005515
mflodmancc3d4422017-08-03 08:27:51 -07005516 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005517 fake_encoder_.SetQualityScaling(true);
5518}
5519
Åsa Persson139f4dc2019-08-02 09:29:58 +02005520TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005521 webrtc::test::ScopedKeyValueConfig field_trials(
5522 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005523 "WebRTC-Video-QualityScalerSettings/"
5524 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5525 // Reset encoder for field trials to take effect.
5526 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005527 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5528 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005529 const int kWidth = 640;
5530 const int kHeight = 360;
5531
Henrik Boström381d1092020-05-12 18:49:07 +02005532 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005533 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005534 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5535 // Frame should not be dropped.
5536 WaitForEncodedFrame(1);
5537
Henrik Boström381d1092020-05-12 18:49:07 +02005538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005539 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5540 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5541 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005542 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5543 // Frame should not be dropped.
5544 WaitForEncodedFrame(2);
5545
Henrik Boström381d1092020-05-12 18:49:07 +02005546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005547 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5548 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5549 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005550 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5551 // Expect to drop this frame, the wait should time out.
5552 ExpectDroppedFrame();
5553
5554 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005555 EXPECT_TRUE_WAIT(
5556 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005557 video_stream_encoder_->Stop();
5558}
5559
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005560TEST_F(VideoStreamEncoderTest,
5561 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005562 webrtc::test::ScopedKeyValueConfig field_trials(
5563 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005564 "WebRTC-Video-QualityScalerSettings/"
5565 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5566 fake_encoder_.SetQualityScaling(false);
5567 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005568 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5569 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005570 const int kWidth = 640;
5571 const int kHeight = 360;
5572
5573 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005574 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005575 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5576 // Frame should not be dropped.
5577 WaitForEncodedFrame(1);
5578
5579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5580 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5581 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5582 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5583 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5584 // Frame should not be dropped.
5585 WaitForEncodedFrame(2);
5586
5587 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5588 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5589 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5590 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5591 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5592 // Not dropped since quality scaling is disabled.
5593 WaitForEncodedFrame(3);
5594
5595 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005596 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005597 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5598
5599 video_stream_encoder_->Stop();
5600}
5601
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005602TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005603 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005604 // Set simulcast.
5605 ResetEncoder("VP8", 3, 1, 1, false);
5606 fake_encoder_.SetQualityScaling(true);
5607 const int kWidth = 1280;
5608 const int kHeight = 720;
5609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005610 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005611 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5612 // Frame should not be dropped.
5613 WaitForEncodedFrame(1);
5614
5615 // Trigger QVGA "singlecast"
5616 // Update the config.
5617 VideoEncoderConfig video_encoder_config;
5618 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5619 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005620 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005621 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005622 "VP8", /*max qp*/ 56, /*screencast*/ false,
5623 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005624 for (auto& layer : video_encoder_config.simulcast_layers) {
5625 layer.num_temporal_layers = 1;
5626 layer.max_framerate = kDefaultFramerate;
5627 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005628 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005629 video_encoder_config.content_type =
5630 VideoEncoderConfig::ContentType::kRealtimeVideo;
5631
5632 video_encoder_config.simulcast_layers[0].active = true;
5633 video_encoder_config.simulcast_layers[1].active = false;
5634 video_encoder_config.simulcast_layers[2].active = false;
5635
5636 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5637 kMaxPayloadLength);
5638 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5639
5640 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5641 // Frame should not be dropped.
5642 WaitForEncodedFrame(2);
5643
5644 // Trigger HD "singlecast"
5645 video_encoder_config.simulcast_layers[0].active = false;
5646 video_encoder_config.simulcast_layers[1].active = false;
5647 video_encoder_config.simulcast_layers[2].active = true;
5648
5649 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5650 kMaxPayloadLength);
5651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5652
5653 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5654 // Frame should be dropped because of initial frame drop.
5655 ExpectDroppedFrame();
5656
5657 // Expect the sink_wants to specify a scaled frame.
5658 EXPECT_TRUE_WAIT(
5659 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5660 video_stream_encoder_->Stop();
5661}
5662
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005663TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005664 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005665 // Set simulcast.
5666 ResetEncoder("VP9", 1, 1, 3, false);
5667 fake_encoder_.SetQualityScaling(true);
5668 const int kWidth = 1280;
5669 const int kHeight = 720;
5670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005671 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005672 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5673 // Frame should not be dropped.
5674 WaitForEncodedFrame(1);
5675
5676 // Trigger QVGA "singlecast"
5677 // Update the config.
5678 VideoEncoderConfig video_encoder_config;
5679 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5680 &video_encoder_config);
5681 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5682 vp9_settings.numberOfSpatialLayers = 3;
5683 // Since only one layer is active - automatic resize should be enabled.
5684 vp9_settings.automaticResizeOn = true;
5685 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005686 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005687 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005688 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005689 video_encoder_config.content_type =
5690 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005691 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005692 // which SVC layers are active.
5693 video_encoder_config.simulcast_layers.resize(3);
5694
5695 video_encoder_config.simulcast_layers[0].active = true;
5696 video_encoder_config.simulcast_layers[1].active = false;
5697 video_encoder_config.simulcast_layers[2].active = false;
5698
5699 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5700 kMaxPayloadLength);
5701 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5702
5703 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5704 // Frame should not be dropped.
5705 WaitForEncodedFrame(2);
5706
5707 // Trigger HD "singlecast"
5708 video_encoder_config.simulcast_layers[0].active = false;
5709 video_encoder_config.simulcast_layers[1].active = false;
5710 video_encoder_config.simulcast_layers[2].active = true;
5711
5712 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5713 kMaxPayloadLength);
5714 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5715
5716 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5717 // Frame should be dropped because of initial frame drop.
5718 ExpectDroppedFrame();
5719
5720 // Expect the sink_wants to specify a scaled frame.
5721 EXPECT_TRUE_WAIT(
5722 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5723 video_stream_encoder_->Stop();
5724}
5725
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005726TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005727 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5728 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5729 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5730 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5731 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5732 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5733 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5734 fake_encoder_.SetResolutionBitrateLimits(
5735 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5736
5737 VideoEncoderConfig video_encoder_config;
5738 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5739 &video_encoder_config);
5740 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5741 vp9_settings.numberOfSpatialLayers = 3;
5742 // Since only one layer is active - automatic resize should be enabled.
5743 vp9_settings.automaticResizeOn = true;
5744 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005745 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005746 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005747 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005748 video_encoder_config.content_type =
5749 VideoEncoderConfig::ContentType::kRealtimeVideo;
5750 // Simulcast layers are used to indicate which spatial layers are active.
5751 video_encoder_config.simulcast_layers.resize(3);
5752 video_encoder_config.simulcast_layers[0].active = false;
5753 video_encoder_config.simulcast_layers[1].active = true;
5754 video_encoder_config.simulcast_layers[2].active = false;
5755
5756 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5757 kMaxPayloadLength);
5758 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5759
5760 // The encoder bitrate limits for 360p should be used.
5761 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5762 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005763 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5764 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5765 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5766 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5767 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5768 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005769 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005770 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005771 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005772 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005773
5774 // The encoder bitrate limits for 270p should be used.
5775 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5776 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005777 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5778 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5779 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5780 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5781 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5782 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005783 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005784 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005785 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005786 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005787
5788 video_stream_encoder_->Stop();
5789}
5790
5791TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005792 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5793 VideoEncoderConfig video_encoder_config;
5794 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5795 &video_encoder_config);
5796 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5797 vp9_settings.numberOfSpatialLayers = 3;
5798 // Since only one layer is active - automatic resize should be enabled.
5799 vp9_settings.automaticResizeOn = true;
5800 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005801 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005802 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005804 video_encoder_config.content_type =
5805 VideoEncoderConfig::ContentType::kRealtimeVideo;
5806 // Simulcast layers are used to indicate which spatial layers are active.
5807 video_encoder_config.simulcast_layers.resize(3);
5808 video_encoder_config.simulcast_layers[0].active = false;
5809 video_encoder_config.simulcast_layers[1].active = true;
5810 video_encoder_config.simulcast_layers[2].active = false;
5811
5812 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5813 kMaxPayloadLength);
5814 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5815
5816 // The default bitrate limits for 360p should be used.
5817 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005818 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5819 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005820 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5821 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005822 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5823 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5824 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5825 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5826 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5827 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005828 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005829 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005830 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005831 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005832
5833 // The default bitrate limits for 270p should be used.
5834 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005835 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5836 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005837 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5838 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005839 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5840 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5841 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5842 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5843 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5844 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005845 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005846 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005847 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005848 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005849
5850 video_stream_encoder_->Stop();
5851}
5852
5853TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005854 webrtc::test::ScopedKeyValueConfig field_trials(
5855 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005856 VideoEncoderConfig video_encoder_config;
5857 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5858 &video_encoder_config);
5859 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5860 vp9_settings.numberOfSpatialLayers = 3;
5861 // Since only one layer is active - automatic resize should be enabled.
5862 vp9_settings.automaticResizeOn = true;
5863 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005864 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005865 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005866 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005867 video_encoder_config.content_type =
5868 VideoEncoderConfig::ContentType::kRealtimeVideo;
5869 // Simulcast layers are used to indicate which spatial layers are active.
5870 video_encoder_config.simulcast_layers.resize(3);
5871 video_encoder_config.simulcast_layers[0].active = false;
5872 video_encoder_config.simulcast_layers[1].active = true;
5873 video_encoder_config.simulcast_layers[2].active = false;
5874
5875 // Reset encoder for field trials to take effect.
5876 ConfigureEncoder(video_encoder_config.Copy());
5877
5878 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5879 kMaxPayloadLength);
5880 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5881
5882 // The default bitrate limits for 360p should not be used.
5883 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005884 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5885 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005886 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5887 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005888 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5889 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5890 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5891 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5892 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5893 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005894 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005895 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005896
5897 video_stream_encoder_->Stop();
5898}
5899
5900TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5901 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5902 /*num_spatial_layers=*/1, /*screenshare=*/false);
5903
5904 // The default singlecast bitrate limits for 720p should not be used.
5905 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005906 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5907 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005908 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5909 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005910 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5911 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5912 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5913 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5914 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5915 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005916 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005917 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005918
5919 video_stream_encoder_->Stop();
5920}
5921
5922TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005923 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5924 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5925 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5926 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5927 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5928 fake_encoder_.SetResolutionBitrateLimits(
5929 {kEncoderLimits180p, kEncoderLimits720p});
5930
5931 VideoEncoderConfig video_encoder_config;
5932 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5933 &video_encoder_config);
5934 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5935 vp9_settings.numberOfSpatialLayers = 3;
5936 // Since only one layer is active - automatic resize should be enabled.
5937 vp9_settings.automaticResizeOn = true;
5938 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005939 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005940 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005941 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005942 video_encoder_config.content_type =
5943 VideoEncoderConfig::ContentType::kRealtimeVideo;
5944 // Simulcast layers are used to indicate which spatial layers are active.
5945 video_encoder_config.simulcast_layers.resize(3);
5946 video_encoder_config.simulcast_layers[0].active = true;
5947 video_encoder_config.simulcast_layers[1].active = false;
5948 video_encoder_config.simulcast_layers[2].active = false;
5949
5950 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5951 kMaxPayloadLength);
5952 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5953
5954 // Limits not applied on lowest stream, limits for 180p should not be used.
5955 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5956 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005957 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5958 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5959 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5960 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5961 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5962 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005963 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005964 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005965 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005966 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005967
5968 video_stream_encoder_->Stop();
5969}
5970
5971TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005972 InitialFrameDropActivatesWhenResolutionIncreases) {
5973 const int kWidth = 640;
5974 const int kHeight = 360;
5975
5976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005977 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005978 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5979 // Frame should not be dropped.
5980 WaitForEncodedFrame(1);
5981
5982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005983 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005984 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5985 // Frame should not be dropped, bitrate not too low for frame.
5986 WaitForEncodedFrame(2);
5987
5988 // Incoming resolution increases.
5989 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5990 // Expect to drop this frame, bitrate too low for frame.
5991 ExpectDroppedFrame();
5992
5993 // Expect the sink_wants to specify a scaled frame.
5994 EXPECT_TRUE_WAIT(
5995 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5996 video_stream_encoder_->Stop();
5997}
5998
5999TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6000 const int kWidth = 640;
6001 const int kHeight = 360;
6002 // So that quality scaling doesn't happen by itself.
6003 fake_encoder_.SetQp(kQpHigh);
6004
6005 AdaptingFrameForwarder source(&time_controller_);
6006 source.set_adaptation_enabled(true);
6007 video_stream_encoder_->SetSource(
6008 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6009
6010 int timestamp = 1;
6011
6012 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006013 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006014 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6015 WaitForEncodedFrame(timestamp);
6016 timestamp += 9000;
6017 // Long pause to disable all first BWE drop logic.
6018 AdvanceTime(TimeDelta::Millis(1000));
6019
6020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006021 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006022 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6023 // Not dropped frame, as initial frame drop is disabled by now.
6024 WaitForEncodedFrame(timestamp);
6025 timestamp += 9000;
6026 AdvanceTime(TimeDelta::Millis(100));
6027
6028 // Quality adaptation down.
6029 video_stream_encoder_->TriggerQualityLow();
6030
6031 // Adaptation has an effect.
6032 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6033 5000);
6034
6035 // Frame isn't dropped as initial frame dropper is disabled.
6036 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6037 WaitForEncodedFrame(timestamp);
6038 timestamp += 9000;
6039 AdvanceTime(TimeDelta::Millis(100));
6040
6041 // Quality adaptation up.
6042 video_stream_encoder_->TriggerQualityHigh();
6043
6044 // Adaptation has an effect.
6045 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6046 5000);
6047
6048 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6049 // Frame should not be dropped, as initial framedropper is off.
6050 WaitForEncodedFrame(timestamp);
6051
6052 video_stream_encoder_->Stop();
6053}
6054
Åsa Persson7f354f82021-02-04 15:52:15 +01006055TEST_F(VideoStreamEncoderTest,
6056 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6057 const int kMinStartBps360p = 222000;
6058 fake_encoder_.SetResolutionBitrateLimits(
6059 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6060 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6061 800000)});
6062
6063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6064 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6065 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6066 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6067 0, 0, 0);
6068 // Frame should not be dropped, bitrate not too low for frame.
6069 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6070 WaitForEncodedFrame(1);
6071
6072 // Incoming resolution increases, initial frame drop activates.
6073 // Frame should be dropped, link allocation too low for frame.
6074 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6075 ExpectDroppedFrame();
6076
6077 // Expect sink_wants to specify a scaled frame.
6078 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6079 5000);
6080 video_stream_encoder_->Stop();
6081}
6082
6083TEST_F(VideoStreamEncoderTest,
6084 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6085 const int kMinStartBps360p = 222000;
6086 fake_encoder_.SetResolutionBitrateLimits(
6087 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6088 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6089 800000)});
6090
6091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6092 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6093 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6094 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6095 0, 0, 0);
6096 // Frame should not be dropped, bitrate not too low for frame.
6097 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6098 WaitForEncodedFrame(1);
6099
6100 // Incoming resolution increases, initial frame drop activates.
6101 // Frame should be dropped, link allocation not too low for frame.
6102 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6103 WaitForEncodedFrame(2);
6104
6105 video_stream_encoder_->Stop();
6106}
6107
Åsa Perssone644a032019-11-08 15:56:00 +01006108TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006109 webrtc::test::ScopedKeyValueConfig field_trials(
6110 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006111 "WebRTC-Video-QualityRampupSettings/"
6112 "min_pixels:921600,min_duration_ms:2000/");
6113
6114 const int kWidth = 1280;
6115 const int kHeight = 720;
6116 const int kFps = 10;
6117 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006118
6119 // Reset encoder for field trials to take effect.
6120 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006121 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006122 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006123 ConfigureEncoder(std::move(config));
6124 fake_encoder_.SetQp(kQpLow);
6125
6126 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006127 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006128 source.set_adaptation_enabled(true);
6129 video_stream_encoder_->SetSource(&source,
6130 DegradationPreference::MAINTAIN_FRAMERATE);
6131
6132 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006133 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006135 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006136
6137 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006138 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006139 int64_t timestamp_ms = kFrameIntervalMs;
6140 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6141 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006142 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6143 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006144
6145 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6147 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006148
Artem Titovab30d722021-07-27 16:22:11 +02006149 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006150 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006151 for (size_t i = 1; i <= 10; i++) {
6152 timestamp_ms += kFrameIntervalMs;
6153 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6154 WaitForEncodedFrame(timestamp_ms);
6155 }
Åsa Persson06defc42021-09-10 15:28:48 +02006156
6157 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6158 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6159 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6160 timestamp_ms += kFrameIntervalMs;
6161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6162 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006163 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6164 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6165
Åsa Persson06defc42021-09-10 15:28:48 +02006166 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006167 timestamp_ms += kFrameIntervalMs;
6168 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6169 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006170 // The ramp-up code involves the adaptation queue, give it time to execute.
6171 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006172 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006173 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006174
6175 // Frame should not be adapted.
6176 timestamp_ms += kFrameIntervalMs;
6177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6178 WaitForEncodedFrame(kWidth, kHeight);
6179 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6180
6181 video_stream_encoder_->Stop();
6182}
6183
mflodmancc3d4422017-08-03 08:27:51 -07006184TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006185 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006186 webrtc::test::ScopedKeyValueConfig field_trials(
6187 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006188 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006189 source.set_adaptation_enabled(true);
6190 video_stream_encoder_->SetSource(&source,
6191 DegradationPreference::MAINTAIN_FRAMERATE);
6192 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006193 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006194 fake_encoder_.SetQp(kQpHigh + 1);
6195 const int kWidth = 1280;
6196 const int kHeight = 720;
6197 const int64_t kFrameIntervalMs = 100;
6198 int64_t timestamp_ms = kFrameIntervalMs;
6199 for (size_t i = 1; i <= 100; i++) {
6200 timestamp_ms += kFrameIntervalMs;
6201 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6202 WaitForEncodedFrame(timestamp_ms);
6203 }
6204 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6205 // for the first time.
6206 // TODO(eshr): We should avoid these waits by using threads with simulated
6207 // time.
6208 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6209 2000 * 2.5 * 2);
6210 timestamp_ms += kFrameIntervalMs;
6211 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6212 WaitForEncodedFrame(timestamp_ms);
6213 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6214 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6215 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6216
6217 // Disable Quality scaling by turning off scaler on the encoder and
6218 // reconfiguring.
6219 fake_encoder_.SetQualityScaling(false);
6220 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6221 kMaxPayloadLength);
6222 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006223 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006224 // Since we turned off the quality scaler, the adaptations made by it are
6225 // removed.
6226 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6227 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6228
6229 video_stream_encoder_->Stop();
6230}
6231
6232TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006233 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6234 const int kTooSmallWidth = 10;
6235 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006236 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006237 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006238
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006239 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006240 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006241 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006242 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006243 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006244 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6245
6246 // Trigger adapt down, too small frame, expect no change.
6247 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006248 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006249 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006250 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006251 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6252 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6253
mflodmancc3d4422017-08-03 08:27:51 -07006254 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006255}
6256
mflodmancc3d4422017-08-03 08:27:51 -07006257TEST_F(VideoStreamEncoderTest,
6258 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006259 const int kTooSmallWidth = 10;
6260 const int kTooSmallHeight = 10;
6261 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006263 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006264
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006265 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006266 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006267 video_stream_encoder_->SetSource(&source,
6268 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006269 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6271 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6272
6273 // Trigger adapt down, expect limited framerate.
6274 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006275 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006276 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006277 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006278 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6279 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6280 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6281
6282 // Trigger adapt down, too small frame, expect no change.
6283 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006284 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006285 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006286 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6288 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6289 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6290
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006292}
6293
mflodmancc3d4422017-08-03 08:27:51 -07006294TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006295 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006297 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006298 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006299 const int kFrameWidth = 1280;
6300 const int kFrameHeight = 720;
6301 video_source_.IncomingCapturedFrame(
6302 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006303 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006304 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006305}
6306
sprangb1ca0732017-02-01 08:38:12 -08006307// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006308TEST_F(VideoStreamEncoderTest,
6309 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006311 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006312
6313 const int kFrameWidth = 1280;
6314 const int kFrameHeight = 720;
6315 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006316 // requested by
6317 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006318 video_source_.set_adaptation_enabled(true);
6319
6320 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006321 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006322 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006323
6324 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006325 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006326 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006327 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006328 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006329
asaperssonfab67072017-04-04 05:51:49 -07006330 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006331 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006332 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006333 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006334 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006335
mflodmancc3d4422017-08-03 08:27:51 -07006336 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006337}
sprangfe627f32017-03-29 08:24:59 -07006338
mflodmancc3d4422017-08-03 08:27:51 -07006339TEST_F(VideoStreamEncoderTest,
6340 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006341 const int kFrameWidth = 1280;
6342 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006343
Henrik Boström381d1092020-05-12 18:49:07 +02006344 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006345 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006346 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006347 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006348 video_source_.set_adaptation_enabled(true);
6349
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006350 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006351
6352 video_source_.IncomingCapturedFrame(
6353 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006354 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006355
6356 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006357 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006358
6359 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006360 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006361 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006362 video_source_.IncomingCapturedFrame(
6363 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006364 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006365 }
6366
6367 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006368 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006369 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006370 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006371 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006372 video_source_.IncomingCapturedFrame(
6373 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006374 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006375 ++num_frames_dropped;
6376 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006377 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006378 }
6379 }
6380
sprang4847ae62017-06-27 07:06:52 -07006381 // Add some slack to account for frames dropped by the frame dropper.
6382 const int kErrorMargin = 1;
6383 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006384 kErrorMargin);
6385
6386 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006387 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006388 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006389 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006390 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006391 video_source_.IncomingCapturedFrame(
6392 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006393 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006394 ++num_frames_dropped;
6395 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006396 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006397 }
6398 }
sprang4847ae62017-06-27 07:06:52 -07006399 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006400 kErrorMargin);
6401
6402 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006403 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006404 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006405 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006406 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006407 video_source_.IncomingCapturedFrame(
6408 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006409 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006410 ++num_frames_dropped;
6411 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006412 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006413 }
6414 }
sprang4847ae62017-06-27 07:06:52 -07006415 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006416 kErrorMargin);
6417
6418 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006419 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006420 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006421 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006422 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006423 video_source_.IncomingCapturedFrame(
6424 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006425 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006426 ++num_frames_dropped;
6427 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006428 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006429 }
6430 }
6431 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6432
mflodmancc3d4422017-08-03 08:27:51 -07006433 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006434}
6435
mflodmancc3d4422017-08-03 08:27:51 -07006436TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006437 const int kFramerateFps = 5;
6438 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006439 const int kFrameWidth = 1280;
6440 const int kFrameHeight = 720;
6441
sprang4847ae62017-06-27 07:06:52 -07006442 // Reconfigure encoder with two temporal layers and screensharing, which will
6443 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006444 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006445
Henrik Boström381d1092020-05-12 18:49:07 +02006446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006447 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006448 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006449 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006450 video_source_.set_adaptation_enabled(true);
6451
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006452 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006453
6454 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006455 rtc::VideoSinkWants last_wants;
6456 do {
6457 last_wants = video_source_.sink_wants();
6458
sprangc5d62e22017-04-02 23:53:04 -07006459 // Insert frames to get a new fps estimate...
6460 for (int j = 0; j < kFramerateFps; ++j) {
6461 video_source_.IncomingCapturedFrame(
6462 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006463 if (video_source_.last_sent_width()) {
6464 sink_.WaitForEncodedFrame(timestamp_ms);
6465 }
sprangc5d62e22017-04-02 23:53:04 -07006466 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006467 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006468 }
6469 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006470 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006471 } while (video_source_.sink_wants().max_framerate_fps <
6472 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006473
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006474 EXPECT_THAT(video_source_.sink_wants(),
6475 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006476
mflodmancc3d4422017-08-03 08:27:51 -07006477 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006478}
asaperssonf7e294d2017-06-13 23:25:22 -07006479
mflodmancc3d4422017-08-03 08:27:51 -07006480TEST_F(VideoStreamEncoderTest,
6481 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006482 const int kWidth = 1280;
6483 const int kHeight = 720;
6484 const int64_t kFrameIntervalMs = 150;
6485 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006487 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006488
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006489 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006490 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006491 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006492 video_stream_encoder_->SetSource(&source,
6493 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006494 timestamp_ms += kFrameIntervalMs;
6495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006496 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006497 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6500 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6501
6502 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006503 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006504 timestamp_ms += kFrameIntervalMs;
6505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006506 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006507 EXPECT_THAT(source.sink_wants(),
6508 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006509 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6510 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6511 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6512
6513 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006514 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006515 timestamp_ms += kFrameIntervalMs;
6516 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006517 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006518 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006519 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6520 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6521 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6522
6523 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006524 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006525 timestamp_ms += kFrameIntervalMs;
6526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006527 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006528 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6530 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6531 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6532
6533 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006534 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006535 timestamp_ms += kFrameIntervalMs;
6536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006537 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006538 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6540 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6541 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6542
6543 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006544 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006545 timestamp_ms += kFrameIntervalMs;
6546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006547 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006548 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6550 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6551 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6552
6553 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006554 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006555 timestamp_ms += kFrameIntervalMs;
6556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006557 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006558 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6561 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6562
6563 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006564 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006565 timestamp_ms += kFrameIntervalMs;
6566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006567 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006568 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006569 rtc::VideoSinkWants last_wants = source.sink_wants();
6570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6571 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6572 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6573
6574 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006575 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006576 timestamp_ms += kFrameIntervalMs;
6577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006578 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006579 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6582 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6583
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006584 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006585 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006586 timestamp_ms += kFrameIntervalMs;
6587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006588 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006589 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6592 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6593
6594 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006595 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006596 timestamp_ms += kFrameIntervalMs;
6597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006598 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006599 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6602 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6603
6604 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006605 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006606 timestamp_ms += kFrameIntervalMs;
6607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006608 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006609 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6612 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6613
6614 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006615 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006616 timestamp_ms += kFrameIntervalMs;
6617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006618 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006619 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6622 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6623
6624 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006625 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006626 timestamp_ms += kFrameIntervalMs;
6627 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006628 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006629 EXPECT_THAT(source.sink_wants(), FpsMax());
6630 EXPECT_EQ(source.sink_wants().max_pixel_count,
6631 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6634 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6635
6636 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006637 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006638 timestamp_ms += kFrameIntervalMs;
6639 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006640 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006641 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6643 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6644 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6645
Åsa Persson30ab0152019-08-27 12:22:33 +02006646 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006647 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006648 timestamp_ms += kFrameIntervalMs;
6649 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006650 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006651 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006652 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6655 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6656
6657 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006658 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006659 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006660 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6661
mflodmancc3d4422017-08-03 08:27:51 -07006662 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006663}
6664
mflodmancc3d4422017-08-03 08:27:51 -07006665TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006666 const int kWidth = 1280;
6667 const int kHeight = 720;
6668 const int64_t kFrameIntervalMs = 150;
6669 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006671 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006672
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006673 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006674 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006675 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006676 video_stream_encoder_->SetSource(&source,
6677 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006678 timestamp_ms += kFrameIntervalMs;
6679 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006680 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006681 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006682 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6683 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6685 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6686 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6687 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6688
6689 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006690 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006691 timestamp_ms += kFrameIntervalMs;
6692 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006693 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006694 EXPECT_THAT(source.sink_wants(),
6695 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006696 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6697 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6698 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6699 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6700 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6701 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6702
6703 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006704 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006705 timestamp_ms += kFrameIntervalMs;
6706 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006707 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006708 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6711 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6712 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6713 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6714 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6715
6716 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006717 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006718 timestamp_ms += kFrameIntervalMs;
6719 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006720 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006721 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006722 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006723 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6724 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6726 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6727 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6728
Evan Shrubsole64469032020-06-11 10:45:29 +02006729 // Trigger cpu adapt up, expect no change since QP is most limited.
6730 {
6731 // Store current sink wants since we expect no change and if there is no
6732 // change then last_wants() is not updated.
6733 auto previous_sink_wants = source.sink_wants();
6734 video_stream_encoder_->TriggerCpuUnderuse();
6735 timestamp_ms += kFrameIntervalMs;
6736 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6737 WaitForEncodedFrame(timestamp_ms);
6738 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6739 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6740 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6741 }
6742
6743 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6744 video_stream_encoder_->TriggerQualityHigh();
6745 timestamp_ms += kFrameIntervalMs;
6746 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6747 WaitForEncodedFrame(timestamp_ms);
6748 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6749 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6750 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6751 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6753 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6754 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6755
6756 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6757 // expect increased resolution (960x540@30fps).
6758 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006759 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006760 timestamp_ms += kFrameIntervalMs;
6761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006762 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006763 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006764 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6765 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6766 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6768 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006769 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006770
Evan Shrubsole64469032020-06-11 10:45:29 +02006771 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6772 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006773 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006774 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006775 timestamp_ms += kFrameIntervalMs;
6776 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006777 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006778 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006779 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006780 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6782 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6784 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006785 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006786
6787 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006788 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006789 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006790 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006791 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006792
mflodmancc3d4422017-08-03 08:27:51 -07006793 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006794}
6795
mflodmancc3d4422017-08-03 08:27:51 -07006796TEST_F(VideoStreamEncoderTest,
6797 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006798 const int kWidth = 640;
6799 const int kHeight = 360;
6800 const int kFpsLimit = 15;
6801 const int64_t kFrameIntervalMs = 150;
6802 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006803 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006804 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006805
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006806 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006807 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006808 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006809 video_stream_encoder_->SetSource(&source,
6810 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006811 timestamp_ms += kFrameIntervalMs;
6812 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006813 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006815 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6817 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6818 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6819 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6820 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6821
6822 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006823 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006824 timestamp_ms += kFrameIntervalMs;
6825 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006826 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006827 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6830 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6831 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6832 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6833 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6834
6835 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006836 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006837 timestamp_ms += kFrameIntervalMs;
6838 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006839 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006840 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006841 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006842 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6844 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6845 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6846 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6847
Evan Shrubsole64469032020-06-11 10:45:29 +02006848 // Trigger cpu adapt up, expect no change because quality is most limited.
6849 {
6850 auto previous_sink_wants = source.sink_wants();
6851 // Store current sink wants since we expect no change ind if there is no
6852 // change then last__wants() is not updated.
6853 video_stream_encoder_->TriggerCpuUnderuse();
6854 timestamp_ms += kFrameIntervalMs;
6855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6856 WaitForEncodedFrame(timestamp_ms);
6857 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6858 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6859 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6860 }
6861
6862 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6863 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006864 timestamp_ms += kFrameIntervalMs;
6865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006866 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006867 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006868 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6870 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006871 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6872 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6873 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006874
Evan Shrubsole64469032020-06-11 10:45:29 +02006875 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006876 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006877 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006878 timestamp_ms += kFrameIntervalMs;
6879 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006880 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006881 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006882 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6884 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6885 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6886 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006887 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006888
6889 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006890 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006891 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006892 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006893 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006894
mflodmancc3d4422017-08-03 08:27:51 -07006895 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006896}
6897
mflodmancc3d4422017-08-03 08:27:51 -07006898TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006899 const int kFrameWidth = 1920;
6900 const int kFrameHeight = 1080;
6901 // 3/4 of 1920.
6902 const int kAdaptedFrameWidth = 1440;
6903 // 3/4 of 1080 rounded down to multiple of 4.
6904 const int kAdaptedFrameHeight = 808;
6905 const int kFramerate = 24;
6906
Henrik Boström381d1092020-05-12 18:49:07 +02006907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006908 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006909 // Trigger reconfigure encoder (without resetting the entire instance).
6910 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006911 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6912 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006913 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006914 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006915 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006916 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006917 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006918 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006919
6920 video_source_.set_adaptation_enabled(true);
6921
6922 video_source_.IncomingCapturedFrame(
6923 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006924 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006925
6926 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006927 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006928 video_source_.IncomingCapturedFrame(
6929 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006930 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006931
mflodmancc3d4422017-08-03 08:27:51 -07006932 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006933}
6934
mflodmancc3d4422017-08-03 08:27:51 -07006935TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006936 const int kFrameWidth = 1280;
6937 const int kFrameHeight = 720;
6938 const int kLowFps = 2;
6939 const int kHighFps = 30;
6940
Henrik Boström381d1092020-05-12 18:49:07 +02006941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006942 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006943
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006944 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006945 max_framerate_ = kLowFps;
6946
6947 // Insert 2 seconds of 2fps video.
6948 for (int i = 0; i < kLowFps * 2; ++i) {
6949 video_source_.IncomingCapturedFrame(
6950 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6951 WaitForEncodedFrame(timestamp_ms);
6952 timestamp_ms += 1000 / kLowFps;
6953 }
6954
6955 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006957 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006958 video_source_.IncomingCapturedFrame(
6959 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6960 WaitForEncodedFrame(timestamp_ms);
6961 timestamp_ms += 1000 / kLowFps;
6962
6963 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6964
6965 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006966 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006967 const int kFrameIntervalMs = 1000 / kHighFps;
6968 max_framerate_ = kHighFps;
6969 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6970 video_source_.IncomingCapturedFrame(
6971 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6972 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6973 // be dropped if the encoder hans't been updated with the new higher target
6974 // framerate yet, causing it to overshoot the target bitrate and then
6975 // suffering the wrath of the media optimizer.
6976 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6977 timestamp_ms += kFrameIntervalMs;
6978 }
6979
6980 // Don expect correct measurement just yet, but it should be higher than
6981 // before.
6982 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6983
mflodmancc3d4422017-08-03 08:27:51 -07006984 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006985}
6986
mflodmancc3d4422017-08-03 08:27:51 -07006987TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006988 const int kFrameWidth = 1280;
6989 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006990 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006991 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006992 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006993
Henrik Boström381d1092020-05-12 18:49:07 +02006994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006996 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006997
6998 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006999 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007000 video_source_.IncomingCapturedFrame(
7001 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7002 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007003 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007004
7005 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007007 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007008
7009 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007010 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007011 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007012
Per Kjellanderdcef6412020-10-07 15:09:05 +02007013 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007014 video_source_.IncomingCapturedFrame(
7015 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7016 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007017 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007018
mflodmancc3d4422017-08-03 08:27:51 -07007019 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007020}
ilnik6b826ef2017-06-16 06:53:48 -07007021
Niels Möller4db138e2018-04-19 09:04:13 +02007022TEST_F(VideoStreamEncoderTest,
7023 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7024 const int kFrameWidth = 1280;
7025 const int kFrameHeight = 720;
7026 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007028 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007029 video_source_.IncomingCapturedFrame(
7030 CreateFrame(1, kFrameWidth, kFrameHeight));
7031 WaitForEncodedFrame(1);
7032 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7033 .low_encode_usage_threshold_percent,
7034 default_options.low_encode_usage_threshold_percent);
7035 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7036 .high_encode_usage_threshold_percent,
7037 default_options.high_encode_usage_threshold_percent);
7038 video_stream_encoder_->Stop();
7039}
7040
7041TEST_F(VideoStreamEncoderTest,
7042 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7043 const int kFrameWidth = 1280;
7044 const int kFrameHeight = 720;
7045 CpuOveruseOptions hardware_options;
7046 hardware_options.low_encode_usage_threshold_percent = 150;
7047 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007048 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007049
Henrik Boström381d1092020-05-12 18:49:07 +02007050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007051 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007052 video_source_.IncomingCapturedFrame(
7053 CreateFrame(1, kFrameWidth, kFrameHeight));
7054 WaitForEncodedFrame(1);
7055 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7056 .low_encode_usage_threshold_percent,
7057 hardware_options.low_encode_usage_threshold_percent);
7058 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7059 .high_encode_usage_threshold_percent,
7060 hardware_options.high_encode_usage_threshold_percent);
7061 video_stream_encoder_->Stop();
7062}
7063
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007064TEST_F(VideoStreamEncoderTest,
7065 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7066 const int kFrameWidth = 1280;
7067 const int kFrameHeight = 720;
7068
7069 const CpuOveruseOptions default_options;
7070 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007071 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007072 video_source_.IncomingCapturedFrame(
7073 CreateFrame(1, kFrameWidth, kFrameHeight));
7074 WaitForEncodedFrame(1);
7075 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7076 .low_encode_usage_threshold_percent,
7077 default_options.low_encode_usage_threshold_percent);
7078 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7079 .high_encode_usage_threshold_percent,
7080 default_options.high_encode_usage_threshold_percent);
7081
7082 CpuOveruseOptions hardware_options;
7083 hardware_options.low_encode_usage_threshold_percent = 150;
7084 hardware_options.high_encode_usage_threshold_percent = 200;
7085 fake_encoder_.SetIsHardwareAccelerated(true);
7086
7087 video_source_.IncomingCapturedFrame(
7088 CreateFrame(2, kFrameWidth, kFrameHeight));
7089 WaitForEncodedFrame(2);
7090
7091 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7092 .low_encode_usage_threshold_percent,
7093 hardware_options.low_encode_usage_threshold_percent);
7094 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7095 .high_encode_usage_threshold_percent,
7096 hardware_options.high_encode_usage_threshold_percent);
7097
7098 video_stream_encoder_->Stop();
7099}
7100
Niels Möller6bb5ab92019-01-11 11:11:10 +01007101TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7102 const int kFrameWidth = 320;
7103 const int kFrameHeight = 240;
7104 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007105 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007106 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7107
Henrik Boström381d1092020-05-12 18:49:07 +02007108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007109 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007110
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007111 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007112 max_framerate_ = kFps;
7113
7114 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7115 fake_encoder_.SimulateOvershoot(1.0);
7116 int num_dropped = 0;
7117 for (int i = 0; i < kNumFramesInRun; ++i) {
7118 video_source_.IncomingCapturedFrame(
7119 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7120 // Wait up to two frame durations for a frame to arrive.
7121 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7122 ++num_dropped;
7123 }
7124 timestamp_ms += 1000 / kFps;
7125 }
7126
Erik Språnga8d48ab2019-02-08 14:17:40 +01007127 // Framerate should be measured to be near the expected target rate.
7128 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7129
7130 // Frame drops should be within 5% of expected 0%.
7131 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007132
7133 // Make encoder produce frames at double the expected bitrate during 3 seconds
7134 // of video, verify number of drops. Rate needs to be slightly changed in
7135 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007136 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007137 const RateControlSettings trials =
7138 RateControlSettings::ParseFromFieldTrials();
7139 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007140 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007141 // frame dropping since the adjuter will try to just lower the target
7142 // bitrate rather than drop frames. If network headroom can be used, it
7143 // doesn't push back as hard so we don't need quite as much overshoot.
7144 // These numbers are unfortunately a bit magical but there's not trivial
7145 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007146 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007147 }
7148 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007149 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007150 kTargetBitrate + DataRate::KilobitsPerSec(1),
7151 kTargetBitrate + DataRate::KilobitsPerSec(1),
7152 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007153 num_dropped = 0;
7154 for (int i = 0; i < kNumFramesInRun; ++i) {
7155 video_source_.IncomingCapturedFrame(
7156 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7157 // Wait up to two frame durations for a frame to arrive.
7158 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7159 ++num_dropped;
7160 }
7161 timestamp_ms += 1000 / kFps;
7162 }
7163
Henrik Boström381d1092020-05-12 18:49:07 +02007164 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007165 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007166
7167 // Target framerate should be still be near the expected target, despite
7168 // the frame drops.
7169 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7170
7171 // Frame drops should be within 5% of expected 50%.
7172 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007173
7174 video_stream_encoder_->Stop();
7175}
7176
7177TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7178 const int kFrameWidth = 320;
7179 const int kFrameHeight = 240;
7180 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007181 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007182
7183 ASSERT_GT(max_framerate_, kActualInputFps);
7184
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007185 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007186 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007187 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007188 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007189
7190 // Insert 3 seconds of video, with an input fps lower than configured max.
7191 for (int i = 0; i < kActualInputFps * 3; ++i) {
7192 video_source_.IncomingCapturedFrame(
7193 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7194 // Wait up to two frame durations for a frame to arrive.
7195 WaitForEncodedFrame(timestamp_ms);
7196 timestamp_ms += 1000 / kActualInputFps;
7197 }
7198
7199 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7200
7201 video_stream_encoder_->Stop();
7202}
7203
Markus Handell9a478b52021-11-18 16:07:01 +01007204TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007205 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007206 test::FrameForwarder source;
7207 video_stream_encoder_->SetSource(&source,
7208 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007210 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007211
Markus Handell9a478b52021-11-18 16:07:01 +01007212 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007213 WaitForEncodedFrame(1);
7214 // On the very first frame full update should be forced.
7215 rect = fake_encoder_.GetLastUpdateRect();
7216 EXPECT_EQ(rect.offset_x, 0);
7217 EXPECT_EQ(rect.offset_y, 0);
7218 EXPECT_EQ(rect.height, codec_height_);
7219 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007220 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7221 // scheduled for processing during encoder queue processing of frame 2.
7222 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7223 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007224 WaitForEncodedFrame(3);
7225 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7226 rect = fake_encoder_.GetLastUpdateRect();
7227 EXPECT_EQ(rect.offset_x, 1);
7228 EXPECT_EQ(rect.offset_y, 0);
7229 EXPECT_EQ(rect.width, 10);
7230 EXPECT_EQ(rect.height, 1);
7231
Markus Handell9a478b52021-11-18 16:07:01 +01007232 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007233 WaitForEncodedFrame(4);
7234 // Previous frame was encoded, so no accumulation should happen.
7235 rect = fake_encoder_.GetLastUpdateRect();
7236 EXPECT_EQ(rect.offset_x, 0);
7237 EXPECT_EQ(rect.offset_y, 0);
7238 EXPECT_EQ(rect.width, 1);
7239 EXPECT_EQ(rect.height, 1);
7240
7241 video_stream_encoder_->Stop();
7242}
7243
Erik Språngd7329ca2019-02-21 21:19:53 +01007244TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007245 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007246 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007247
7248 // First frame is always keyframe.
7249 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7250 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007251 EXPECT_THAT(
7252 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007253 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007254
7255 // Insert delta frame.
7256 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7257 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007258 EXPECT_THAT(
7259 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007260 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007261
7262 // Request next frame be a key-frame.
7263 video_stream_encoder_->SendKeyFrame();
7264 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7265 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007266 EXPECT_THAT(
7267 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007268 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007269
7270 video_stream_encoder_->Stop();
7271}
7272
7273TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7274 // Setup simulcast with three streams.
7275 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007277 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7278 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007279 // Wait for all three layers before triggering event.
7280 sink_.SetNumExpectedLayers(3);
7281
7282 // First frame is always keyframe.
7283 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7284 WaitForEncodedFrame(1);
7285 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007286 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7287 VideoFrameType::kVideoFrameKey,
7288 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007289
7290 // Insert delta frame.
7291 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7292 WaitForEncodedFrame(2);
7293 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007294 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7295 VideoFrameType::kVideoFrameDelta,
7296 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007297
7298 // Request next frame be a key-frame.
7299 // Only first stream is configured to produce key-frame.
7300 video_stream_encoder_->SendKeyFrame();
7301 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7302 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007303
7304 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7305 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007306 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007307 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007308 VideoFrameType::kVideoFrameKey,
7309 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007310
7311 video_stream_encoder_->Stop();
7312}
7313
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007314TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007315 // SPS contains VUI with restrictions on the maximum number of reordered
7316 // pictures, there is no need to rewrite the bitstream to enable faster
7317 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007318 ResetEncoder("H264", 1, 1, 1, false);
7319
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007321 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007322 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007323
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007324 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007325 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007326
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007327 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7328 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007329
7330 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007331 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007332
7333 video_stream_encoder_->Stop();
7334}
7335
7336TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007337 // SPS does not contain VUI, the bitstream is will be rewritten with added
7338 // VUI with restrictions on the maximum number of reordered pictures to
7339 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007340 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7341 0x00, 0x00, 0x03, 0x03, 0xF4,
7342 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007343 ResetEncoder("H264", 1, 1, 1, false);
7344
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007346 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007347 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007348
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007349 fake_encoder_.SetEncodedImageData(
7350 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007351
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007352 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7353 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007354
7355 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007356 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007357
7358 video_stream_encoder_->Stop();
7359}
7360
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007361TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7362 const int kFrameWidth = 1280;
7363 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007364 const DataRate kTargetBitrate =
7365 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007366
Henrik Boström381d1092020-05-12 18:49:07 +02007367 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007368 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007369 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7370
7371 // Insert a first video frame. It should be dropped because of downscale in
7372 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007373 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007374 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7375 frame.set_rotation(kVideoRotation_270);
7376 video_source_.IncomingCapturedFrame(frame);
7377
7378 ExpectDroppedFrame();
7379
7380 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007381 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007382 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7383 frame.set_rotation(kVideoRotation_90);
7384 video_source_.IncomingCapturedFrame(frame);
7385
7386 WaitForEncodedFrame(timestamp_ms);
7387 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7388
7389 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007390 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007391 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7392 frame.set_rotation(kVideoRotation_180);
7393 video_source_.IncomingCapturedFrame(frame);
7394
7395 WaitForEncodedFrame(timestamp_ms);
7396 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7397
7398 video_stream_encoder_->Stop();
7399}
7400
Erik Språng5056af02019-09-02 15:53:11 +02007401TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7402 const int kFrameWidth = 320;
7403 const int kFrameHeight = 180;
7404
7405 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007407 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7408 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7409 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007410 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007411 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007412 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007413
7414 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007415 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007416 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7417 frame.set_rotation(kVideoRotation_270);
7418 video_source_.IncomingCapturedFrame(frame);
7419 WaitForEncodedFrame(timestamp_ms);
7420
7421 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007422 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007423 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7424 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007425 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007426 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007427 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007428 /*link_allocation=*/target_rate,
7429 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007430 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007431 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007432 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7433
7434 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7435 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7436 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007437 DataRate allocation_sum =
7438 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007439 EXPECT_EQ(min_rate, allocation_sum);
7440 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7441
7442 video_stream_encoder_->Stop();
7443}
7444
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007445TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007447 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007448 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007449 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007450 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7451 WaitForEncodedFrame(1);
7452
7453 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7454 ASSERT_TRUE(prev_rate_settings.has_value());
7455 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7456 kDefaultFramerate);
7457
7458 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7459 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7460 timestamp_ms += 1000 / kDefaultFramerate;
7461 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7462 WaitForEncodedFrame(timestamp_ms);
7463 }
7464 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7465 kDefaultFramerate);
7466 // Capture larger frame to trigger a reconfigure.
7467 codec_height_ *= 2;
7468 codec_width_ *= 2;
7469 timestamp_ms += 1000 / kDefaultFramerate;
7470 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7471 WaitForEncodedFrame(timestamp_ms);
7472
7473 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7474 auto current_rate_settings =
7475 fake_encoder_.GetAndResetLastRateControlSettings();
7476 // Ensure we have actually reconfigured twice
7477 // The rate settings should have been set again even though
7478 // they haven't changed.
7479 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007480 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007481
7482 video_stream_encoder_->Stop();
7483}
7484
philipeld9cc8c02019-09-16 14:53:40 +02007485struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007486 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007487 MOCK_METHOD(void,
7488 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007489 (const webrtc::SdpVideoFormat& format,
7490 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007491 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007492};
7493
philipel9b058032020-02-10 11:30:00 +01007494TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7495 constexpr int kDontCare = 100;
7496 StrictMock<MockEncoderSelector> encoder_selector;
7497 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7498 &fake_encoder_, &encoder_selector);
7499 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7500
7501 // Reset encoder for new configuration to take effect.
7502 ConfigureEncoder(video_encoder_config_.Copy());
7503
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007504 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007505
7506 video_source_.IncomingCapturedFrame(
7507 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007508 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007509 video_stream_encoder_->Stop();
7510
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007511 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007512 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007513 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7514 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007515 video_stream_encoder_.reset();
7516}
7517
7518TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7519 constexpr int kDontCare = 100;
7520
7521 NiceMock<MockEncoderSelector> encoder_selector;
7522 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7523 video_send_config_.encoder_settings.encoder_switch_request_callback =
7524 &switch_callback;
7525 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7526 &fake_encoder_, &encoder_selector);
7527 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7528
7529 // Reset encoder for new configuration to take effect.
7530 ConfigureEncoder(video_encoder_config_.Copy());
7531
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007532 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007533 .WillByDefault(Return(SdpVideoFormat("AV1")));
7534 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007535 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7536 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007537
Henrik Boström381d1092020-05-12 18:49:07 +02007538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007539 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7540 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7541 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007542 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007543 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007544 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007545 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007546
7547 video_stream_encoder_->Stop();
7548}
7549
7550TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7551 constexpr int kSufficientBitrateToNotDrop = 1000;
7552 constexpr int kDontCare = 100;
7553
7554 NiceMock<MockVideoEncoder> video_encoder;
7555 NiceMock<MockEncoderSelector> encoder_selector;
7556 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7557 video_send_config_.encoder_settings.encoder_switch_request_callback =
7558 &switch_callback;
7559 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7560 &video_encoder, &encoder_selector);
7561 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7562
7563 // Reset encoder for new configuration to take effect.
7564 ConfigureEncoder(video_encoder_config_.Copy());
7565
7566 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7567 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7568 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007570 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7571 /*stable_target_bitrate=*/
7572 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7573 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007574 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007575 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007576 /*cwnd_reduce_ratio=*/0);
7577
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007578 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007579 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007580 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007581 .WillByDefault(Return(SdpVideoFormat("AV2")));
7582
7583 rtc::Event encode_attempted;
7584 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007585 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7586 /*allow_default_fallback=*/true))
7587 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007588
7589 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7590 encode_attempted.Wait(3000);
7591
Markus Handell28c71802021-11-08 10:11:55 +01007592 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007593
philipel9b058032020-02-10 11:30:00 +01007594 video_stream_encoder_->Stop();
7595
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007596 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7597 // to it's factory, so in order for the encoder instance in the
7598 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7599 // reset the `video_stream_encoder_` here.
7600 video_stream_encoder_.reset();
7601}
7602
7603TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7604 NiceMock<MockVideoEncoder> video_encoder;
7605 NiceMock<MockEncoderSelector> encoder_selector;
7606 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7607 video_send_config_.encoder_settings.encoder_switch_request_callback =
7608 &switch_callback;
7609 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7610 &video_encoder, &encoder_selector);
7611 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7612
7613 // Reset encoder for new configuration to take effect.
7614 ConfigureEncoder(video_encoder_config_.Copy());
7615
7616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7617 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7618 /*round_trip_time_ms=*/0,
7619 /*cwnd_reduce_ratio=*/0);
7620 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7621
7622 ON_CALL(video_encoder, InitEncode(_, _))
7623 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7624 ON_CALL(encoder_selector, OnEncoderBroken)
7625 .WillByDefault(Return(SdpVideoFormat("AV2")));
7626
7627 rtc::Event encode_attempted;
7628 EXPECT_CALL(switch_callback,
7629 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7630 /*allow_default_fallback=*/true))
7631 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7632
7633 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7634 encode_attempted.Wait(3000);
7635
7636 AdvanceTime(TimeDelta::Zero());
7637
7638 video_stream_encoder_->Stop();
7639
7640 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7641 // to it's factory, so in order for the encoder instance in the
7642 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7643 // reset the `video_stream_encoder_` here.
7644 video_stream_encoder_.reset();
7645}
7646
7647TEST_F(VideoStreamEncoderTest,
7648 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7649 NiceMock<MockVideoEncoder> video_encoder;
7650 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7651 video_send_config_.encoder_settings.encoder_switch_request_callback =
7652 &switch_callback;
7653 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7654 &video_encoder, /*encoder_selector=*/nullptr);
7655 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7656
7657 // Reset encoder for new configuration to take effect.
7658 ConfigureEncoder(video_encoder_config_.Copy());
7659
7660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7661 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7662 /*round_trip_time_ms=*/0,
7663 /*cwnd_reduce_ratio=*/0);
7664 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7665
7666 ON_CALL(video_encoder, InitEncode(_, _))
7667 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7668
7669 rtc::Event encode_attempted;
7670 EXPECT_CALL(switch_callback,
7671 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7672 /*allow_default_fallback=*/true))
7673 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7674
7675 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7676 encode_attempted.Wait(3000);
7677
7678 AdvanceTime(TimeDelta::Zero());
7679
7680 video_stream_encoder_->Stop();
7681
7682 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007683 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007684 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7685 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007686 video_stream_encoder_.reset();
7687}
7688
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007689TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007690 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007691 const int kFrameWidth = 320;
7692 const int kFrameHeight = 180;
7693
7694 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007695 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007697 /*target_bitrate=*/rate,
7698 /*stable_target_bitrate=*/rate,
7699 /*link_allocation=*/rate,
7700 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007701 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007702 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007703
7704 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007705 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007706 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7707 frame.set_rotation(kVideoRotation_270);
7708 video_source_.IncomingCapturedFrame(frame);
7709 WaitForEncodedFrame(timestamp_ms);
7710 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7711
7712 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007713 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007715 /*target_bitrate=*/new_stable_rate,
7716 /*stable_target_bitrate=*/new_stable_rate,
7717 /*link_allocation=*/rate,
7718 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007719 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007720 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007721 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7722 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7723 video_stream_encoder_->Stop();
7724}
7725
7726TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007727 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007728 const int kFrameWidth = 320;
7729 const int kFrameHeight = 180;
7730
7731 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007732 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007734 /*target_bitrate=*/rate,
7735 /*stable_target_bitrate=*/rate,
7736 /*link_allocation=*/rate,
7737 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007738 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007739 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007740
7741 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007742 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007743 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7744 frame.set_rotation(kVideoRotation_270);
7745 video_source_.IncomingCapturedFrame(frame);
7746 WaitForEncodedFrame(timestamp_ms);
7747 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7748
7749 // Set a higher target rate without changing the link_allocation. Should not
7750 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007751 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007753 /*target_bitrate=*/rate,
7754 /*stable_target_bitrate=*/new_stable_rate,
7755 /*link_allocation=*/rate,
7756 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007757 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007758 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007759 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7760 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7761 video_stream_encoder_->Stop();
7762}
7763
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007764TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007765 test::ScopedKeyValueConfig field_trials(
7766 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007767 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7768 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7769 const int kFramerateFps = 30;
7770 const int kWidth = 1920;
7771 const int kHeight = 1080;
7772 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7773 // Works on screenshare mode.
7774 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7775 // We rely on the automatic resolution adaptation, but we handle framerate
7776 // adaptation manually by mocking the stats proxy.
7777 video_source_.set_adaptation_enabled(true);
7778
7779 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007781 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007782 video_stream_encoder_->SetSource(&video_source_,
7783 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007784 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007785
7786 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7787 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7788
7789 // Pass enough frames with the full update to trigger animation detection.
7790 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007791 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007792 frame.set_ntp_time_ms(timestamp_ms);
7793 frame.set_timestamp_us(timestamp_ms * 1000);
7794 video_source_.IncomingCapturedFrame(frame);
7795 WaitForEncodedFrame(timestamp_ms);
7796 }
7797
7798 // Resolution should be limited.
7799 rtc::VideoSinkWants expected;
7800 expected.max_framerate_fps = kFramerateFps;
7801 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007802 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007803
7804 // Pass one frame with no known update.
7805 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007806 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007807 frame.set_ntp_time_ms(timestamp_ms);
7808 frame.set_timestamp_us(timestamp_ms * 1000);
7809 frame.clear_update_rect();
7810
7811 video_source_.IncomingCapturedFrame(frame);
7812 WaitForEncodedFrame(timestamp_ms);
7813
7814 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007815 EXPECT_THAT(video_source_.sink_wants(),
7816 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007817
7818 video_stream_encoder_->Stop();
7819}
7820
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007821TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7822 const int kWidth = 720; // 540p adapted down.
7823 const int kHeight = 405;
7824 const int kNumFrames = 3;
7825 // Works on screenshare mode.
7826 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7827 /*num_spatial_layers=*/2, /*screenshare=*/true);
7828
7829 video_source_.set_adaptation_enabled(true);
7830
7831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007833
7834 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7835
7836 // Pass enough frames with the full update to trigger animation detection.
7837 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007838 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007839 frame.set_ntp_time_ms(timestamp_ms);
7840 frame.set_timestamp_us(timestamp_ms * 1000);
7841 video_source_.IncomingCapturedFrame(frame);
7842 WaitForEncodedFrame(timestamp_ms);
7843 }
7844
7845 video_stream_encoder_->Stop();
7846}
7847
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007848TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7849 const float downscale_factors[] = {4.0, 2.0, 1.0};
7850 const int number_layers =
7851 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7852 VideoEncoderConfig config;
7853 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7854 for (int i = 0; i < number_layers; ++i) {
7855 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7856 config.simulcast_layers[i].active = true;
7857 }
7858 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007859 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007860 "VP8", /*max qp*/ 56, /*screencast*/ false,
7861 /*screenshare enabled*/ false);
7862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007863 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7864 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007865
7866 // First initialization.
7867 // Encoder should be initialized. Next frame should be key frame.
7868 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7869 sink_.SetNumExpectedLayers(number_layers);
7870 int64_t timestamp_ms = kFrameIntervalMs;
7871 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7872 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007873 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007874 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7875 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7876 VideoFrameType::kVideoFrameKey,
7877 VideoFrameType::kVideoFrameKey}));
7878
7879 // Disable top layer.
7880 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7881 config.simulcast_layers[number_layers - 1].active = false;
7882 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7883 sink_.SetNumExpectedLayers(number_layers - 1);
7884 timestamp_ms += kFrameIntervalMs;
7885 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7886 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007887 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007888 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7889 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7890 VideoFrameType::kVideoFrameDelta,
7891 VideoFrameType::kVideoFrameDelta}));
7892
7893 // Re-enable top layer.
7894 // Encoder should be re-initialized. Next frame should be key frame.
7895 config.simulcast_layers[number_layers - 1].active = true;
7896 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7897 sink_.SetNumExpectedLayers(number_layers);
7898 timestamp_ms += kFrameIntervalMs;
7899 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7900 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007901 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007902 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7903 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7904 VideoFrameType::kVideoFrameKey,
7905 VideoFrameType::kVideoFrameKey}));
7906
7907 // Top layer max rate change.
7908 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7909 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7910 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7911 sink_.SetNumExpectedLayers(number_layers);
7912 timestamp_ms += kFrameIntervalMs;
7913 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7914 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007915 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007916 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7917 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7918 VideoFrameType::kVideoFrameDelta,
7919 VideoFrameType::kVideoFrameDelta}));
7920
7921 // Top layer resolution change.
7922 // Encoder should be re-initialized. Next frame should be key frame.
7923 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7924 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7925 sink_.SetNumExpectedLayers(number_layers);
7926 timestamp_ms += kFrameIntervalMs;
7927 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7928 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007929 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007930 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7931 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7932 VideoFrameType::kVideoFrameKey,
7933 VideoFrameType::kVideoFrameKey}));
7934 video_stream_encoder_->Stop();
7935}
7936
Henrik Boström1124ed12021-02-25 10:30:39 +01007937TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7938 const int kFrameWidth = 1280;
7939 const int kFrameHeight = 720;
7940
7941 SetUp();
7942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007943 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007944
7945 // Capturing a frame should reconfigure the encoder and expose the encoder
7946 // resolution, which is the same as the input frame.
7947 int64_t timestamp_ms = kFrameIntervalMs;
7948 video_source_.IncomingCapturedFrame(
7949 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7950 WaitForEncodedFrame(timestamp_ms);
7951 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7952 EXPECT_THAT(video_source_.sink_wants().resolutions,
7953 ::testing::ElementsAreArray(
7954 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7955
7956 video_stream_encoder_->Stop();
7957}
7958
7959TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7960 // Pick downscale factors such that we never encode at full resolution - this
7961 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007962 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007963 // encoder should not ask for the frame resolution. This allows video frames
7964 // to have the appearence of one resolution but optimize its internal buffers
7965 // for what is actually encoded.
7966 const size_t kNumSimulcastLayers = 3u;
7967 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7968 const int kFrameWidth = 1280;
7969 const int kFrameHeight = 720;
7970 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7971 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7972 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7973 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7974 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7975 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7976
7977 VideoEncoderConfig config;
7978 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7979 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7980 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7981 config.simulcast_layers[i].active = true;
7982 }
7983 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007984 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007985 "VP8", /*max qp*/ 56, /*screencast*/ false,
7986 /*screenshare enabled*/ false);
7987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007988 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7989 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007990
7991 // Capture a frame with all layers active.
7992 int64_t timestamp_ms = kFrameIntervalMs;
7993 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7994 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7995 video_source_.IncomingCapturedFrame(
7996 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7997 WaitForEncodedFrame(timestamp_ms);
7998 // Expect encoded resolutions to match the expected simulcast layers.
7999 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8000 EXPECT_THAT(
8001 video_source_.sink_wants().resolutions,
8002 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8003
8004 // Capture a frame with one of the layers inactive.
8005 timestamp_ms += kFrameIntervalMs;
8006 config.simulcast_layers[2].active = false;
8007 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8008 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8009 video_source_.IncomingCapturedFrame(
8010 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8011 WaitForEncodedFrame(timestamp_ms);
8012
8013 // Expect encoded resolutions to match the expected simulcast layers.
8014 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8015 EXPECT_THAT(video_source_.sink_wants().resolutions,
8016 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8017
8018 // Capture a frame with all but one layer turned off.
8019 timestamp_ms += kFrameIntervalMs;
8020 config.simulcast_layers[1].active = false;
8021 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8022 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8023 video_source_.IncomingCapturedFrame(
8024 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8025 WaitForEncodedFrame(timestamp_ms);
8026
8027 // Expect encoded resolutions to match the expected simulcast layers.
8028 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8029 EXPECT_THAT(video_source_.sink_wants().resolutions,
8030 ::testing::ElementsAreArray({kLayer0Size}));
8031
8032 video_stream_encoder_->Stop();
8033}
8034
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008035TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008036 ResetEncoder("VP8", 1, 1, 1, false);
8037
Niels Möller8b692902021-06-14 12:04:57 +02008038 // Force encoder reconfig.
8039 video_source_.IncomingCapturedFrame(
8040 CreateFrame(1, codec_width_, codec_height_));
8041 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8042
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008043 // Set QP on encoded frame and pass the frame to encode complete callback.
8044 // Since QP is present QP parsing won't be triggered and the original value
8045 // should be kept.
8046 EncodedImage encoded_image;
8047 encoded_image.qp_ = 123;
8048 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8049 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8050 CodecSpecificInfo codec_info;
8051 codec_info.codecType = kVideoCodecVP8;
8052 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8053 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8054 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8055 video_stream_encoder_->Stop();
8056}
8057
8058TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008059 ResetEncoder("VP8", 1, 1, 1, false);
8060
Niels Möller8b692902021-06-14 12:04:57 +02008061 // Force encoder reconfig.
8062 video_source_.IncomingCapturedFrame(
8063 CreateFrame(1, codec_width_, codec_height_));
8064 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8065
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008066 // Pass an encoded frame without QP to encode complete callback. QP should be
8067 // parsed and set.
8068 EncodedImage encoded_image;
8069 encoded_image.qp_ = -1;
8070 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8071 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8072 CodecSpecificInfo codec_info;
8073 codec_info.codecType = kVideoCodecVP8;
8074 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8075 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8076 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8077 video_stream_encoder_->Stop();
8078}
8079
8080TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008081 webrtc::test::ScopedKeyValueConfig field_trials(
8082 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008083
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008084 ResetEncoder("VP8", 1, 1, 1, false);
8085
Niels Möller8b692902021-06-14 12:04:57 +02008086 // Force encoder reconfig.
8087 video_source_.IncomingCapturedFrame(
8088 CreateFrame(1, codec_width_, codec_height_));
8089 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8090
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008091 EncodedImage encoded_image;
8092 encoded_image.qp_ = -1;
8093 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8094 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8095 CodecSpecificInfo codec_info;
8096 codec_info.codecType = kVideoCodecVP8;
8097 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8098 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8099 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8100 video_stream_encoder_->Stop();
8101}
8102
Sergey Silkind19e3b92021-03-16 10:05:30 +00008103TEST_F(VideoStreamEncoderTest,
8104 QualityScalingNotAllowed_QualityScalingDisabled) {
8105 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8106
8107 // Disable scaling settings in encoder info.
8108 fake_encoder_.SetQualityScaling(false);
8109 // Disable quality scaling in encoder config.
8110 video_encoder_config.is_quality_scaling_allowed = false;
8111 ConfigureEncoder(std::move(video_encoder_config));
8112
8113 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008114 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008115
8116 test::FrameForwarder source;
8117 video_stream_encoder_->SetSource(
8118 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8119 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8120 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8121
8122 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8123 WaitForEncodedFrame(1);
8124 video_stream_encoder_->TriggerQualityLow();
8125 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8126
8127 video_stream_encoder_->Stop();
8128}
8129
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008130TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8131 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8132
8133 // Disable scaling settings in encoder info.
8134 fake_encoder_.SetQualityScaling(false);
8135 // Set QP trusted in encoder info.
8136 fake_encoder_.SetIsQpTrusted(true);
8137 // Enable quality scaling in encoder config.
8138 video_encoder_config.is_quality_scaling_allowed = false;
8139 ConfigureEncoder(std::move(video_encoder_config));
8140
8141 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008142 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008143
8144 test::FrameForwarder source;
8145 video_stream_encoder_->SetSource(
8146 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8147 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8148 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8149
8150 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8151 WaitForEncodedFrame(1);
8152 video_stream_encoder_->TriggerQualityLow();
8153 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8154
8155 video_stream_encoder_->Stop();
8156}
8157
Shuhai Pengf2707702021-09-29 17:19:44 +08008158TEST_F(VideoStreamEncoderTest,
8159 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8160 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8161
8162 // Disable scaling settings in encoder info.
8163 fake_encoder_.SetQualityScaling(false);
8164 // Set QP trusted in encoder info.
8165 fake_encoder_.SetIsQpTrusted(true);
8166 // Enable quality scaling in encoder config.
8167 video_encoder_config.is_quality_scaling_allowed = false;
8168 ConfigureEncoder(std::move(video_encoder_config));
8169
8170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008171 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008172
8173 test::FrameForwarder source;
8174 video_stream_encoder_->SetSource(
8175 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8176 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8177 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8178
8179 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8180 WaitForEncodedFrame(1);
8181 video_stream_encoder_->TriggerQualityLow();
8182 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8183
8184 video_stream_encoder_->Stop();
8185}
8186
8187TEST_F(VideoStreamEncoderTest,
8188 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8189 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8190
8191 // Disable scaling settings in encoder info.
8192 fake_encoder_.SetQualityScaling(false);
8193 // Set QP trusted in encoder info.
8194 fake_encoder_.SetIsQpTrusted(false);
8195 // Enable quality scaling in encoder config.
8196 video_encoder_config.is_quality_scaling_allowed = false;
8197 ConfigureEncoder(std::move(video_encoder_config));
8198
8199 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008200 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008201
8202 test::FrameForwarder source;
8203 video_stream_encoder_->SetSource(
8204 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8205 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8206 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8207
8208 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8209 WaitForEncodedFrame(1);
8210 video_stream_encoder_->TriggerQualityLow();
8211 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8212
8213 video_stream_encoder_->Stop();
8214}
8215
8216TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8217 // Set QP trusted in encoder info.
8218 fake_encoder_.SetIsQpTrusted(false);
8219
8220 const int MinEncBitrateKbps = 30;
8221 const int MaxEncBitrateKbps = 100;
8222 const int MinStartBitrateKbp = 50;
8223 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8224 /*frame_size_pixels=*/codec_width_ * codec_height_,
8225 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8226 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8227 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8228
8229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008230 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008231
8232 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8233
8234 VideoEncoderConfig video_encoder_config;
8235 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8236 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8237 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8238 MinEncBitrateKbps * 1000;
8239 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8240 kMaxPayloadLength);
8241
8242 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8243 WaitForEncodedFrame(1);
8244 EXPECT_EQ(
8245 MaxEncBitrateKbps,
8246 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8247 EXPECT_EQ(
8248 MinEncBitrateKbps,
8249 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8250
8251 video_stream_encoder_->Stop();
8252}
8253
8254TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8255 // Set QP trusted in encoder info.
8256 fake_encoder_.SetIsQpTrusted(false);
8257
8258 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8259 EncoderInfoSettings::
8260 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8261 codec_width_ * codec_height_,
8262 EncoderInfoSettings::
8263 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8264 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8265
8266 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8267 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8268 const int TargetEncBitrate = MaxEncBitrate;
8269 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8270 DataRate::BitsPerSec(TargetEncBitrate),
8271 DataRate::BitsPerSec(TargetEncBitrate),
8272 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8273
8274 VideoEncoderConfig video_encoder_config;
8275 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8276 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8277 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8278 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8279 kMaxPayloadLength);
8280
8281 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8282 WaitForEncodedFrame(1);
8283 EXPECT_EQ(
8284 MaxEncBitrate / 1000,
8285 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8286 EXPECT_EQ(
8287 MinEncBitrate / 1000,
8288 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8289
8290 video_stream_encoder_->Stop();
8291}
8292
Sergey Silkind19e3b92021-03-16 10:05:30 +00008293#if !defined(WEBRTC_IOS)
8294// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8295// disabled by default on iOS.
8296TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8297 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8298
8299 // Disable scaling settings in encoder info.
8300 fake_encoder_.SetQualityScaling(false);
8301 // Enable quality scaling in encoder config.
8302 video_encoder_config.is_quality_scaling_allowed = true;
8303 ConfigureEncoder(std::move(video_encoder_config));
8304
8305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008306 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008307
8308 test::FrameForwarder source;
8309 video_stream_encoder_->SetSource(
8310 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8311 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8312 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8313
8314 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8315 WaitForEncodedFrame(1);
8316 video_stream_encoder_->TriggerQualityLow();
8317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8318
8319 video_stream_encoder_->Stop();
8320}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008321
8322TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8323 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8324
8325 // Disable scaling settings in encoder info.
8326 fake_encoder_.SetQualityScaling(false);
8327 // Set QP trusted in encoder info.
8328 fake_encoder_.SetIsQpTrusted(true);
8329 // Enable quality scaling in encoder config.
8330 video_encoder_config.is_quality_scaling_allowed = true;
8331 ConfigureEncoder(std::move(video_encoder_config));
8332
8333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008334 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008335
8336 test::FrameForwarder source;
8337 video_stream_encoder_->SetSource(
8338 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8339 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8340 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8341
8342 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8343 WaitForEncodedFrame(1);
8344 video_stream_encoder_->TriggerQualityLow();
8345 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8346
8347 video_stream_encoder_->Stop();
8348}
Shuhai Pengf2707702021-09-29 17:19:44 +08008349
8350TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8351 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8352
8353 // Disable scaling settings in encoder info.
8354 fake_encoder_.SetQualityScaling(false);
8355 // Set QP not trusted in encoder info.
8356 fake_encoder_.SetIsQpTrusted(false);
8357 // Enable quality scaling in encoder config.
8358 video_encoder_config.is_quality_scaling_allowed = true;
8359 ConfigureEncoder(std::move(video_encoder_config));
8360
8361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008362 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008363
8364 test::FrameForwarder source;
8365 video_stream_encoder_->SetSource(
8366 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8367 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8368 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8369
8370 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8371 WaitForEncodedFrame(1);
8372 video_stream_encoder_->TriggerQualityLow();
8373 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8374 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8375 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8376
8377 video_stream_encoder_->Stop();
8378}
8379
8380TEST_F(VideoStreamEncoderTest,
8381 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8382 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8383
8384 // Disable scaling settings in encoder info.
8385 fake_encoder_.SetQualityScaling(false);
8386 // Set QP trusted in encoder info.
8387 fake_encoder_.SetIsQpTrusted(true);
8388 // Enable quality scaling in encoder config.
8389 video_encoder_config.is_quality_scaling_allowed = true;
8390 ConfigureEncoder(std::move(video_encoder_config));
8391
8392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008393 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008394
8395 test::FrameForwarder source;
8396 video_stream_encoder_->SetSource(
8397 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8398 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8399 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8400
8401 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8402 WaitForEncodedFrame(1);
8403 video_stream_encoder_->TriggerQualityLow();
8404 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8405 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8406
8407 video_stream_encoder_->Stop();
8408}
8409
8410TEST_F(VideoStreamEncoderTest,
8411 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8412 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8413
8414 // Disable scaling settings in encoder info.
8415 fake_encoder_.SetQualityScaling(false);
8416 // Set QP trusted in encoder info.
8417 fake_encoder_.SetIsQpTrusted(false);
8418 // Enable quality scaling in encoder config.
8419 video_encoder_config.is_quality_scaling_allowed = true;
8420 ConfigureEncoder(std::move(video_encoder_config));
8421
8422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008423 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008424
8425 test::FrameForwarder source;
8426 video_stream_encoder_->SetSource(
8427 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8428 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8430
8431 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8432 WaitForEncodedFrame(1);
8433 video_stream_encoder_->TriggerQualityLow();
8434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8435
8436 video_stream_encoder_->Stop();
8437}
8438
Sergey Silkind19e3b92021-03-16 10:05:30 +00008439#endif
8440
Henrik Boström56db9ff2021-03-24 09:06:45 +01008441// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8442class VideoStreamEncoderWithRealEncoderTest
8443 : public VideoStreamEncoderTest,
8444 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8445 public:
8446 VideoStreamEncoderWithRealEncoderTest()
8447 : VideoStreamEncoderTest(),
8448 codec_type_(std::get<0>(GetParam())),
8449 allow_i420_conversion_(std::get<1>(GetParam())) {}
8450
8451 void SetUp() override {
8452 VideoStreamEncoderTest::SetUp();
8453 std::unique_ptr<VideoEncoder> encoder;
8454 switch (codec_type_) {
8455 case kVideoCodecVP8:
8456 encoder = VP8Encoder::Create();
8457 break;
8458 case kVideoCodecVP9:
8459 encoder = VP9Encoder::Create();
8460 break;
8461 case kVideoCodecAV1:
philipel95701502022-01-18 18:47:52 +01008462 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008463 break;
8464 case kVideoCodecH264:
8465 encoder =
8466 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8467 break;
8468 case kVideoCodecMultiplex:
8469 mock_encoder_factory_for_multiplex_ =
8470 std::make_unique<MockVideoEncoderFactory>();
8471 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8472 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8473 .WillRepeatedly([] { return VP8Encoder::Create(); });
8474 encoder = std::make_unique<MultiplexEncoderAdapter>(
8475 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8476 false);
8477 break;
8478 default:
Artem Titovd3251962021-11-15 16:57:07 +01008479 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008480 }
8481 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8482 }
8483
8484 void TearDown() override {
8485 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008486 // Ensure `video_stream_encoder_` is destroyed before
8487 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008488 video_stream_encoder_.reset();
8489 VideoStreamEncoderTest::TearDown();
8490 }
8491
8492 protected:
8493 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8494 std::unique_ptr<VideoEncoder> encoder) {
8495 // Configure VSE to use the encoder.
8496 encoder_ = std::move(encoder);
8497 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8498 encoder_.get(), &encoder_selector_);
8499 video_send_config_.encoder_settings.encoder_factory =
8500 encoder_proxy_factory_.get();
8501 VideoEncoderConfig video_encoder_config;
8502 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8503 video_encoder_config_ = video_encoder_config.Copy();
8504 ConfigureEncoder(video_encoder_config_.Copy());
8505
8506 // Set bitrate to ensure frame is not dropped.
8507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008508 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008509 }
8510
8511 const VideoCodecType codec_type_;
8512 const bool allow_i420_conversion_;
8513 NiceMock<MockEncoderSelector> encoder_selector_;
8514 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8515 std::unique_ptr<VideoEncoder> encoder_;
8516 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8517};
8518
8519TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8520 auto native_i420_frame = test::CreateMappableNativeFrame(
8521 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8522 video_source_.IncomingCapturedFrame(native_i420_frame);
8523 WaitForEncodedFrame(codec_width_, codec_height_);
8524
8525 auto mappable_native_buffer =
8526 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8527 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8528 mappable_native_buffer->GetMappedFramedBuffers();
8529 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8530 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8531 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8532 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8533}
8534
8535TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8536 auto native_nv12_frame = test::CreateMappableNativeFrame(
8537 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8538 video_source_.IncomingCapturedFrame(native_nv12_frame);
8539 WaitForEncodedFrame(codec_width_, codec_height_);
8540
8541 auto mappable_native_buffer =
8542 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8543 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8544 mappable_native_buffer->GetMappedFramedBuffers();
8545 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8546 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8547 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8548 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8549
8550 if (!allow_i420_conversion_) {
8551 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8552 }
8553}
8554
Erik Språng7444b192021-06-02 14:02:13 +02008555TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8556 if (codec_type_ == kVideoCodecMultiplex) {
8557 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8558 return;
8559 }
8560
8561 const size_t kNumSpatialLayers = 3u;
8562 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8563 const int kFrameWidth = 1280;
8564 const int kFrameHeight = 720;
8565 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8566 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8567 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8568 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8569 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8570 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8571
8572 VideoEncoderConfig config;
8573 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8574 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008575 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008576 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8577 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8578 vp9_settings.numberOfTemporalLayers = 3;
8579 vp9_settings.automaticResizeOn = false;
8580 config.encoder_specific_settings =
8581 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8582 vp9_settings);
8583 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8584 /*fps=*/30.0,
8585 /*first_active_layer=*/0,
8586 /*num_spatial_layers=*/3,
8587 /*num_temporal_layers=*/3,
8588 /*is_screenshare=*/false);
8589 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8590 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008591 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008592 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8593 /*fps=*/30.0,
8594 /*first_active_layer=*/0,
8595 /*num_spatial_layers=*/3,
8596 /*num_temporal_layers=*/3,
8597 /*is_screenshare=*/false);
8598 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8599 } else {
8600 // Simulcast for VP8/H264.
8601 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8602 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8603 config.simulcast_layers[i].scale_resolution_down_by =
8604 kDownscaleFactors[i];
8605 config.simulcast_layers[i].active = true;
8606 }
8607 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8608 // Turn off frame dropping to prevent flakiness.
8609 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8610 h264_settings.frameDroppingOn = false;
8611 config.encoder_specific_settings = rtc::make_ref_counted<
8612 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8613 }
8614 }
8615
8616 auto set_layer_active = [&](int layer_idx, bool active) {
8617 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8618 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8619 config.spatial_layers[layer_idx].active = active;
8620 } else {
8621 config.simulcast_layers[layer_idx].active = active;
8622 }
8623 };
8624
8625 config.video_stream_factory =
8626 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8627 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8628 /*screencast*/ false,
8629 /*screenshare enabled*/ false);
8630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008631 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8632 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008633
8634 // Capture a frame with all layers active.
8635 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8636 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8637 int64_t timestamp_ms = kFrameIntervalMs;
8638 video_source_.IncomingCapturedFrame(
8639 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8640
8641 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8642 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8643
8644 // Capture a frame with one of the layers inactive.
8645 set_layer_active(2, false);
8646 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8647 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8648 timestamp_ms += kFrameIntervalMs;
8649 video_source_.IncomingCapturedFrame(
8650 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8651 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8652
8653 // New target bitrates signaled based on lower resolution.
8654 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8655 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8656 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8657 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8658
8659 // Re-enable the top layer.
8660 set_layer_active(2, true);
8661 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8662 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8663 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8664
8665 // Bitrate target adjusted back up to enable HD layer...
8666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8667 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8668 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8669 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8670
8671 // ...then add a new frame.
8672 timestamp_ms += kFrameIntervalMs;
8673 video_source_.IncomingCapturedFrame(
8674 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8675 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8676 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8677
8678 video_stream_encoder_->Stop();
8679}
8680
Henrik Boström56db9ff2021-03-24 09:06:45 +01008681std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8682 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8683 VideoCodecType codec_type = std::get<0>(info.param);
8684 bool allow_i420_conversion = std::get<1>(info.param);
8685 std::string str;
8686 switch (codec_type) {
8687 case kVideoCodecGeneric:
8688 str = "Generic";
8689 break;
8690 case kVideoCodecVP8:
8691 str = "VP8";
8692 break;
8693 case kVideoCodecVP9:
8694 str = "VP9";
8695 break;
8696 case kVideoCodecAV1:
8697 str = "AV1";
8698 break;
8699 case kVideoCodecH264:
8700 str = "H264";
8701 break;
8702 case kVideoCodecMultiplex:
8703 str = "Multiplex";
8704 break;
8705 default:
Artem Titovd3251962021-11-15 16:57:07 +01008706 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008707 }
8708 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8709 return str;
8710}
8711
8712constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8713 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8714constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8715 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8716constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Andrey Logvinfef00262022-03-09 11:37:17 +00008717 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008718constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8719 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8720#if defined(WEBRTC_USE_H264)
8721constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8722 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8723
8724// The windows compiler does not tolerate #if statements inside the
8725// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8726// and without H264).
8727INSTANTIATE_TEST_SUITE_P(
8728 All,
8729 VideoStreamEncoderWithRealEncoderTest,
8730 ::testing::Values(kVP8DisallowConversion,
8731 kVP9DisallowConversion,
8732 kAV1AllowConversion,
8733 kMultiplexDisallowConversion,
8734 kH264AllowConversion),
8735 TestParametersVideoCodecAndAllowI420ConversionToString);
8736#else
8737INSTANTIATE_TEST_SUITE_P(
8738 All,
8739 VideoStreamEncoderWithRealEncoderTest,
8740 ::testing::Values(kVP8DisallowConversion,
8741 kVP9DisallowConversion,
8742 kAV1AllowConversion,
8743 kMultiplexDisallowConversion),
8744 TestParametersVideoCodecAndAllowI420ConversionToString);
8745#endif
8746
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008747class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8748 protected:
8749 void RunTest(const std::vector<VideoStream>& configs,
8750 const int expected_num_init_encode) {
8751 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008752 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008753 InsertFrameAndWaitForEncoded();
8754 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8755 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008756 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8757 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008758
8759 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8760 ConfigureEncoder(configs[1]);
8761 InsertFrameAndWaitForEncoded();
8762 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8763 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008764 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008765 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008766 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008767
8768 video_stream_encoder_->Stop();
8769 }
8770
8771 void ConfigureEncoder(const VideoStream& stream) {
8772 VideoEncoderConfig config;
8773 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8774 config.max_bitrate_bps = stream.max_bitrate_bps;
8775 config.simulcast_layers[0] = stream;
8776 config.video_stream_factory =
8777 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8778 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8779 /*conference_mode=*/false);
8780 video_stream_encoder_->ConfigureEncoder(std::move(config),
8781 kMaxPayloadLength);
8782 }
8783
8784 void OnBitrateUpdated(DataRate bitrate) {
8785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8786 bitrate, bitrate, bitrate, 0, 0, 0);
8787 }
8788
8789 void InsertFrameAndWaitForEncoded() {
8790 timestamp_ms_ += kFrameIntervalMs;
8791 video_source_.IncomingCapturedFrame(
8792 CreateFrame(timestamp_ms_, kWidth, kHeight));
8793 sink_.WaitForEncodedFrame(timestamp_ms_);
8794 }
8795
8796 void ExpectEqual(const VideoCodec& actual,
8797 const VideoStream& expected) const {
8798 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8799 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8800 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8801 static_cast<unsigned int>(expected.min_bitrate_bps));
8802 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8803 static_cast<unsigned int>(expected.max_bitrate_bps));
8804 EXPECT_EQ(actual.simulcastStream[0].width,
8805 kWidth / expected.scale_resolution_down_by);
8806 EXPECT_EQ(actual.simulcastStream[0].height,
8807 kHeight / expected.scale_resolution_down_by);
8808 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8809 expected.num_temporal_layers);
8810 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8811 }
8812
8813 VideoStream DefaultConfig() const {
8814 VideoStream stream;
8815 stream.max_framerate = 25;
8816 stream.min_bitrate_bps = 35000;
8817 stream.max_bitrate_bps = 900000;
8818 stream.scale_resolution_down_by = 1.0;
8819 stream.num_temporal_layers = 1;
8820 stream.bitrate_priority = 1.0;
8821 stream.scalability_mode = "";
8822 return stream;
8823 }
8824
8825 const int kWidth = 640;
8826 const int kHeight = 360;
8827 int64_t timestamp_ms_ = 0;
8828};
8829
8830TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8831 VideoStream config1 = DefaultConfig();
8832 VideoStream config2 = config1;
8833 config2.max_framerate++;
8834
8835 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8836}
8837
8838TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8839 VideoStream config1 = DefaultConfig();
8840 VideoStream config2 = config1;
8841 config2.min_bitrate_bps += 10000;
8842
8843 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8844}
8845
8846TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8847 VideoStream config1 = DefaultConfig();
8848 VideoStream config2 = config1;
8849 config2.max_bitrate_bps += 100000;
8850
8851 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8852}
8853
8854TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8855 VideoStream config1 = DefaultConfig();
8856 VideoStream config2 = config1;
8857 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8858
8859 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8860}
8861
8862TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8863 VideoStream config1 = DefaultConfig();
8864 VideoStream config2 = config1;
8865 config2.scale_resolution_down_by *= 2;
8866
8867 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8868}
8869
8870TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8871 VideoStream config1 = DefaultConfig();
8872 VideoStream config2 = config1;
8873 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8874
8875 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8876}
8877
8878TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8879 VideoStream config1 = DefaultConfig();
8880 VideoStream config2 = config1;
8881 config2.scalability_mode = "L1T2";
8882
8883 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8884}
8885
Tommi62b01db2022-01-25 23:41:22 +01008886// Simple test that just creates and then immediately destroys an encoder.
8887// The purpose of the test is to make sure that nothing bad happens if the
8888// initialization step on the encoder queue, doesn't run.
8889TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
8890 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
8891 public:
8892 SuperLazyTaskQueue() = default;
8893 ~SuperLazyTaskQueue() override = default;
8894
8895 private:
8896 void Delete() override { delete this; }
8897 void PostTask(std::unique_ptr<QueuedTask> task) override {
8898 // meh.
8899 }
8900 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
8901 uint32_t milliseconds) override {
8902 ASSERT_TRUE(false);
8903 }
8904 };
8905
8906 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01008907 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 23:41:22 +01008908 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
8909 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
8910 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01008911 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01008912 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
8913 time_controller.GetClock());
8914 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
8915 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
8916 CreateBuiltinVideoBitrateAllocatorFactory();
8917 VideoStreamEncoderSettings encoder_settings{
8918 VideoEncoder::Capabilities(/*loss_notification=*/false)};
8919 encoder_settings.encoder_factory = &encoder_factory;
8920 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
8921
8922 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8923 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
8924
8925 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
8926 encoder_queue(new SuperLazyTaskQueue());
8927
8928 // Construct a VideoStreamEncoder instance and let it go out of scope without
8929 // doing anything else (including calling Stop()). This should be fine since
8930 // the posted init task will simply be deleted.
8931 auto encoder = std::make_unique<VideoStreamEncoder>(
8932 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
8933 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
8934 std::move(adapter), std::move(encoder_queue),
8935 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008936 kVideoBitrateAllocation,
8937 field_trials);
Tommi62b01db2022-01-25 23:41:22 +01008938}
8939
Markus Handellb4e96d42021-11-05 12:00:55 +01008940TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8941 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8942 auto* adapter_ptr = adapter.get();
8943 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01008944 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8945 nullptr;
8946 EXPECT_CALL(*adapter_ptr, Initialize)
8947 .WillOnce(Invoke([&video_stream_encoder_callback](
8948 FrameCadenceAdapterInterface::Callback* callback) {
8949 video_stream_encoder_callback = callback;
8950 }));
8951 TaskQueueBase* encoder_queue = nullptr;
8952 auto video_stream_encoder =
8953 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01008954
Markus Handelle59fee82021-12-23 09:29:23 +01008955 // First a call before we know the frame size and hence cannot compute the
8956 // number of simulcast layers.
8957 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8958 &FrameCadenceAdapterInterface::
8959 ZeroHertzModeParams::num_simulcast_layers,
8960 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01008961 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01008962 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01008963 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8964 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01008965 factory.DepleteTaskQueues();
8966
8967 // Then a call as we've computed the number of simulcast layers after a passed
8968 // frame.
8969 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8970 &FrameCadenceAdapterInterface::
8971 ZeroHertzModeParams::num_simulcast_layers,
8972 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01008973 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01008974 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008975 Mock::VerifyAndClearExpectations(adapter_ptr);
8976
Markus Handelle59fee82021-12-23 09:29:23 +01008977 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01008978 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01008979 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01008980 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01008981 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8982 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008983 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01008984 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008985}
8986
8987TEST(VideoStreamEncoderFrameCadenceTest,
8988 ForwardsFramesIntoFrameCadenceAdapter) {
8989 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8990 auto* adapter_ptr = adapter.get();
8991 test::FrameForwarder video_source;
8992 SimpleVideoStreamEncoderFactory factory;
8993 auto video_stream_encoder = factory.Create(std::move(adapter));
8994 video_stream_encoder->SetSource(
8995 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8996
8997 EXPECT_CALL(*adapter_ptr, OnFrame);
8998 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8999 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009000 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009001}
9002
Markus Handellee225432021-11-29 12:35:12 +01009003TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9004 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9005 auto* adapter_ptr = adapter.get();
9006 test::FrameForwarder video_source;
9007 SimpleVideoStreamEncoderFactory factory;
9008 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9009 nullptr;
9010 EXPECT_CALL(*adapter_ptr, Initialize)
9011 .WillOnce(Invoke([&video_stream_encoder_callback](
9012 FrameCadenceAdapterInterface::Callback* callback) {
9013 video_stream_encoder_callback = callback;
9014 }));
9015 TaskQueueBase* encoder_queue = nullptr;
9016 auto video_stream_encoder =
9017 factory.Create(std::move(adapter), &encoder_queue);
9018
9019 // This is just to make the VSE operational. We'll feed a frame directly by
9020 // the callback interface.
9021 video_stream_encoder->SetSource(
9022 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9023
9024 VideoEncoderConfig video_encoder_config;
9025 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9026 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9027 /*max_data_payload_length=*/1000);
9028
9029 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9030 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009031 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009032 factory.DepleteTaskQueues();
9033}
9034
Markus Handell8d87c462021-12-16 11:37:16 +01009035TEST(VideoStreamEncoderFrameCadenceTest,
9036 DeactivatesActivatesLayersOnBitrateChanges) {
9037 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9038 auto* adapter_ptr = adapter.get();
9039 SimpleVideoStreamEncoderFactory factory;
9040 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9041 nullptr;
9042 EXPECT_CALL(*adapter_ptr, Initialize)
9043 .WillOnce(Invoke([&video_stream_encoder_callback](
9044 FrameCadenceAdapterInterface::Callback* callback) {
9045 video_stream_encoder_callback = callback;
9046 }));
9047 TaskQueueBase* encoder_queue = nullptr;
9048 auto video_stream_encoder =
9049 factory.Create(std::move(adapter), &encoder_queue);
9050
9051 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9052 // {150000, 450000}.
9053 VideoEncoderConfig video_encoder_config;
9054 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9055 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9056 kMaxPayloadLength);
9057 // Ensure an encoder is created.
9058 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9059
9060 // Both layers enabled at 1 MBit/s.
9061 video_stream_encoder->OnBitrateUpdated(
9062 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9063 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9064 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9065 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9066 factory.DepleteTaskQueues();
9067 Mock::VerifyAndClearExpectations(adapter_ptr);
9068
9069 // Layer 1 disabled at 200 KBit/s.
9070 video_stream_encoder->OnBitrateUpdated(
9071 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9072 DataRate::KilobitsPerSec(200), 0, 0, 0);
9073 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9074 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9075 factory.DepleteTaskQueues();
9076 Mock::VerifyAndClearExpectations(adapter_ptr);
9077
9078 // All layers off at suspended video.
9079 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9080 DataRate::Zero(), 0, 0, 0);
9081 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9082 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9083 factory.DepleteTaskQueues();
9084 Mock::VerifyAndClearExpectations(adapter_ptr);
9085
9086 // Both layers enabled again back at 1 MBit/s.
9087 video_stream_encoder->OnBitrateUpdated(
9088 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9089 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9090 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9091 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9092 factory.DepleteTaskQueues();
9093}
9094
9095TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9096 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9097 auto* adapter_ptr = adapter.get();
9098 SimpleVideoStreamEncoderFactory factory;
9099 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9100 nullptr;
9101 EXPECT_CALL(*adapter_ptr, Initialize)
9102 .WillOnce(Invoke([&video_stream_encoder_callback](
9103 FrameCadenceAdapterInterface::Callback* callback) {
9104 video_stream_encoder_callback = callback;
9105 }));
9106 TaskQueueBase* encoder_queue = nullptr;
9107 auto video_stream_encoder =
9108 factory.Create(std::move(adapter), &encoder_queue);
9109
9110 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9111 VideoEncoderConfig video_encoder_config;
9112 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9113 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9114 kMaxPayloadLength);
9115 video_stream_encoder->OnBitrateUpdated(
9116 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9117 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9118
9119 // Pass a frame which has unconverged results.
9120 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9121 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9122 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9123 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9124 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9125 CodecSpecificInfo codec_specific;
9126 codec_specific.codecType = kVideoCodecGeneric;
9127 return codec_specific;
9128 }));
9129 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9130 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9131 factory.DepleteTaskQueues();
9132 Mock::VerifyAndClearExpectations(adapter_ptr);
9133 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9134
9135 // Pass a frame which converges in layer 0 and not in layer 1.
9136 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9137 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9138 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9139 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9140 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9141 CodecSpecificInfo codec_specific;
9142 codec_specific.codecType = kVideoCodecGeneric;
9143 return codec_specific;
9144 }));
9145 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9146 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9147 factory.DepleteTaskQueues();
9148 Mock::VerifyAndClearExpectations(adapter_ptr);
9149 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9150}
9151
Markus Handell2e0f4f02021-12-21 19:14:58 +01009152TEST(VideoStreamEncoderFrameCadenceTest,
9153 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9154 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9155 auto* adapter_ptr = adapter.get();
9156 MockVideoSourceInterface mock_source;
9157 SimpleVideoStreamEncoderFactory factory;
9158 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9159 nullptr;
9160 EXPECT_CALL(*adapter_ptr, Initialize)
9161 .WillOnce(Invoke([&video_stream_encoder_callback](
9162 FrameCadenceAdapterInterface::Callback* callback) {
9163 video_stream_encoder_callback = callback;
9164 }));
9165 TaskQueueBase* encoder_queue = nullptr;
9166 auto video_stream_encoder =
9167 factory.Create(std::move(adapter), &encoder_queue);
9168 video_stream_encoder->SetSource(
9169 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9170 VideoEncoderConfig config;
9171 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9172 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9173 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9174 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9175 // Ensure the encoder is set up.
9176 factory.DepleteTaskQueues();
9177
Markus Handell818e7fb2021-12-30 13:01:33 +01009178 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9179 .WillOnce(Invoke([video_stream_encoder_callback] {
9180 video_stream_encoder_callback->RequestRefreshFrame();
9181 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009182 EXPECT_CALL(mock_source, RequestRefreshFrame);
9183 video_stream_encoder->SendKeyFrame();
9184 factory.DepleteTaskQueues();
9185 Mock::VerifyAndClearExpectations(adapter_ptr);
9186 Mock::VerifyAndClearExpectations(&mock_source);
9187
Markus Handell818e7fb2021-12-30 13:01:33 +01009188 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009189 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9190 video_stream_encoder->SendKeyFrame();
9191 factory.DepleteTaskQueues();
9192}
9193
Markus Handell818e7fb2021-12-30 13:01:33 +01009194TEST(VideoStreamEncoderFrameCadenceTest,
9195 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9196 SimpleVideoStreamEncoderFactory factory;
9197 auto encoder_queue =
9198 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9199 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9200
9201 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009202 test::ScopedKeyValueConfig field_trials(
9203 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009204 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009205 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9206 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009207 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9208
9209 MockVideoSourceInterface mock_source;
9210 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009211 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009212
9213 video_stream_encoder->SetSource(
9214 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9215 VideoEncoderConfig config;
9216 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9217 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9218 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9219
9220 // Eventually expect a refresh frame request when requesting a key frame
9221 // before initializing zero-hertz mode. This can happen in reality because the
9222 // threads invoking key frame requests and constraints setup aren't
9223 // synchronized.
9224 EXPECT_CALL(mock_source, RequestRefreshFrame);
9225 video_stream_encoder->SendKeyFrame();
9226 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9227 factory.DepleteTaskQueues();
9228}
9229
perkj26091b12016-09-01 01:17:40 -07009230} // namespace webrtc